2024-04-22T14:58:54.925 INFO:root:teuthology version: 0.0.1.dev269+g9481b1d 2024-04-22T14:58:54.925 DEBUG:teuthology.run:Teuthology command: teuthology --owner scheduled_adking@teuthology --description orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} --verbose --name adking-2024-04-22_14:50:52-orch:cephadm-wip-adk-testing-2024-04-22-0809-distro-default-smithi --archive /home/teuthworker/archive/adking-2024-04-22_14:50:52-orch:cephadm-wip-adk-testing-2024-04-22-0809-distro-default-smithi/7668335 -- /home/teuthworker/archive/adking-2024-04-22_14:50:52-orch:cephadm-wip-adk-testing-2024-04-22-0809-distro-default-smithi/7668335/orig.config.yaml 2024-04-22T14:58:54.974 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-22T14:58:55.062 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/adking-2024-04-22_14:50:52-orch:cephadm-wip-adk-testing-2024-04-22-0809-distro-default-smithi/7668335 branch: wip-adk-testing-2024-04-22-0809 description: orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} email: adking@redhat.com first_in_suite: false job_id: '7668335' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: adking-2024-04-22_14:50:52-orch:cephadm-wip-adk-testing-2024-04-22-0809-distro-default-smithi no_nested_subset: false nuke-on-error: true os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-adk-testing-2024-04-22-0809 ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mgr/cephadm/use_agent: false mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) log-only-match: - CEPHADM_ sha1: 5883d494d3973b30b3f03e4a3f0e1e06e3c131aa ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 5883d494d3973b30b3f03e4a3f0e1e06e3c131aa workunit: branch: wip-adk-testing-2024-04-22-0809 sha1: 5883d494d3973b30b3f03e4a3f0e1e06e3c131aa owner: scheduled_adking@teuthology priority: 80 repo: https://git.ceph.com/ceph-ci.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b seed: 523 sha1: 5883d494d3973b30b3f03e4a3f0e1e06e3c131aa sleep_before_teardown: 0 subset: 1/16 suite: orch:cephadm suite_branch: wip-adk-testing-2024-04-22-0809 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_5883d494d3973b30b3f03e4a3f0e1e06e3c131aa/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 5883d494d3973b30b3f03e4a3f0e1e06e3c131aa targets: smithi145.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHPdUOGkYO7/OwsFa+YW/UoxoMMcci4O3XA+snK5J2/1FxJyxa7DO4qcJo8QfCaF+BflZXltElTfrzhQxOjy2Ic= smithi172.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPkhKyWiTpQZhRqu8tjjYZYkldYw64E5tTU6Uq1IXcNeUeXTxPfFCXkMueOhIBCMRWNwKdNWDUh9zZJ0kdqZw6U= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 - cephadm.shell: env: - sha1 mon.a: - radosgw-admin realm create --rgw-realm=r --default - radosgw-admin zonegroup create --rgw-zonegroup=default --master --default - radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default - radosgw-admin period update --rgw-realm=r --commit - ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000 - ceph osd pool create foo - rbd pool init foo - ceph orch apply iscsi foo u p - sleep 180 - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph orch ps - ceph versions - ceph -s - ceph orch ls - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 2' - ceph mgr fail - sleep 180 - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph health detail - ceph -s - ceph mgr fail - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 1' - ceph mgr fail - sleep 180 - ceph orch ps - ceph orch ls - ceph versions - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph versions | jq -e '.mgr | length == 1' - ceph versions | jq -e '.mgr | keys' | grep $sha1 - ceph versions | jq -e '.overall | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 1' - ceph versions | jq -e '.mon | keys' | grep $sha1 - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 5' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 7' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 8' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 1' - ceph versions | jq -e '.osd | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.rgw | length == 1' - ceph versions | jq -e '.rgw | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 mon.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph versions - ceph orch upgrade status - ceph health detail - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - ceph orch ls | grep '^osd ' - cephadm.shell: mon.a: - ceph orch upgrade ls - ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0 - ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2 teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17 timestamp: 2024-04-22_14:50:52 tube: smithi user: adking verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-22T14:58:55.062 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_5883d494d3973b30b3f03e4a3f0e1e06e3c131aa/qa; will attempt to use it 2024-04-22T14:58:55.063 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_5883d494d3973b30b3f03e4a3f0e1e06e3c131aa/qa/tasks 2024-04-22T14:58:55.063 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-22T14:58:55.064 INFO:teuthology.task.internal:Checking packages... 2024-04-22T14:58:55.090 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '5883d494d3973b30b3f03e4a3f0e1e06e3c131aa' 2024-04-22T14:58:55.090 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-22T14:58:55.090 INFO:teuthology.packaging:ref: None 2024-04-22T14:58:55.090 INFO:teuthology.packaging:tag: None 2024-04-22T14:58:55.090 INFO:teuthology.packaging:branch: wip-adk-testing-2024-04-22-0809 2024-04-22T14:58:55.090 INFO:teuthology.packaging:sha1: 5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T14:58:55.091 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-adk-testing-2024-04-22-0809 2024-04-22T14:58:55.348 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-3223.g5883d494 2024-04-22T14:58:55.349 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-22T14:58:55.355 INFO:teuthology.task.internal:no buildpackages task found 2024-04-22T14:58:55.355 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-22T14:58:55.362 INFO:teuthology.task.internal:Saving configuration 2024-04-22T14:58:55.380 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-22T14:58:55.385 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-22T14:58:55.407 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi145.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-04-22_14:50:52-orch:cephadm-wip-adk-testing-2024-04-22-0809-distro-default-smithi/7668335', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-22 14:52:15.246176', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHPdUOGkYO7/OwsFa+YW/UoxoMMcci4O3XA+snK5J2/1FxJyxa7DO4qcJo8QfCaF+BflZXltElTfrzhQxOjy2Ic='} 2024-04-22T14:58:55.425 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi172.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-04-22_14:50:52-orch:cephadm-wip-adk-testing-2024-04-22-0809-distro-default-smithi/7668335', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-22 14:52:15.247885', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPkhKyWiTpQZhRqu8tjjYZYkldYw64E5tTU6Uq1IXcNeUeXTxPfFCXkMueOhIBCMRWNwKdNWDUh9zZJ0kdqZw6U='} 2024-04-22T14:58:55.426 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-22T14:58:55.432 INFO:teuthology.task.internal:roles: ubuntu@smithi145.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-22T14:58:55.432 INFO:teuthology.task.internal:roles: ubuntu@smithi172.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-22T14:58:55.432 INFO:teuthology.run_tasks:Running task console_log... 2024-04-22T14:58:55.543 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f90aacac160>, signals=[15]) 2024-04-22T14:58:55.544 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-22T14:58:55.549 INFO:teuthology.task.internal:Opening connections... 2024-04-22T14:58:55.550 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi145.front.sepia.ceph.com 2024-04-22T14:58:55.551 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi145.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-22T14:58:55.627 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi172.front.sepia.ceph.com 2024-04-22T14:58:55.628 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi172.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-22T14:58:55.701 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-22T14:58:55.706 DEBUG:teuthology.orchestra.run.smithi145:> uname -m 2024-04-22T14:58:55.724 INFO:teuthology.orchestra.run.smithi145.stdout:x86_64 2024-04-22T14:58:55.725 DEBUG:teuthology.orchestra.run.smithi145:> cat /etc/os-release 2024-04-22T14:58:55.781 INFO:teuthology.orchestra.run.smithi145.stdout:NAME="CentOS Stream" 2024-04-22T14:58:55.781 INFO:teuthology.orchestra.run.smithi145.stdout:VERSION="9" 2024-04-22T14:58:55.781 INFO:teuthology.orchestra.run.smithi145.stdout:ID="centos" 2024-04-22T14:58:55.781 INFO:teuthology.orchestra.run.smithi145.stdout:ID_LIKE="rhel fedora" 2024-04-22T14:58:55.781 INFO:teuthology.orchestra.run.smithi145.stdout:VERSION_ID="9" 2024-04-22T14:58:55.781 INFO:teuthology.orchestra.run.smithi145.stdout:PLATFORM_ID="platform:el9" 2024-04-22T14:58:55.781 INFO:teuthology.orchestra.run.smithi145.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-22T14:58:55.782 INFO:teuthology.orchestra.run.smithi145.stdout:ANSI_COLOR="0;31" 2024-04-22T14:58:55.782 INFO:teuthology.orchestra.run.smithi145.stdout:LOGO="fedora-logo-icon" 2024-04-22T14:58:55.782 INFO:teuthology.orchestra.run.smithi145.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-22T14:58:55.782 INFO:teuthology.orchestra.run.smithi145.stdout:HOME_URL="https://centos.org/" 2024-04-22T14:58:55.782 INFO:teuthology.orchestra.run.smithi145.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-22T14:58:55.782 INFO:teuthology.orchestra.run.smithi145.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-22T14:58:55.782 INFO:teuthology.orchestra.run.smithi145.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-22T14:58:55.783 INFO:teuthology.lock.ops:Updating smithi145.front.sepia.ceph.com on lock server 2024-04-22T14:58:55.807 DEBUG:teuthology.orchestra.run.smithi172:> uname -m 2024-04-22T14:58:55.825 INFO:teuthology.orchestra.run.smithi172.stdout:x86_64 2024-04-22T14:58:55.825 DEBUG:teuthology.orchestra.run.smithi172:> cat /etc/os-release 2024-04-22T14:58:55.883 INFO:teuthology.orchestra.run.smithi172.stdout:NAME="CentOS Stream" 2024-04-22T14:58:55.883 INFO:teuthology.orchestra.run.smithi172.stdout:VERSION="9" 2024-04-22T14:58:55.883 INFO:teuthology.orchestra.run.smithi172.stdout:ID="centos" 2024-04-22T14:58:55.883 INFO:teuthology.orchestra.run.smithi172.stdout:ID_LIKE="rhel fedora" 2024-04-22T14:58:55.883 INFO:teuthology.orchestra.run.smithi172.stdout:VERSION_ID="9" 2024-04-22T14:58:55.883 INFO:teuthology.orchestra.run.smithi172.stdout:PLATFORM_ID="platform:el9" 2024-04-22T14:58:55.883 INFO:teuthology.orchestra.run.smithi172.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-22T14:58:55.883 INFO:teuthology.orchestra.run.smithi172.stdout:ANSI_COLOR="0;31" 2024-04-22T14:58:55.883 INFO:teuthology.orchestra.run.smithi172.stdout:LOGO="fedora-logo-icon" 2024-04-22T14:58:55.883 INFO:teuthology.orchestra.run.smithi172.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-22T14:58:55.883 INFO:teuthology.orchestra.run.smithi172.stdout:HOME_URL="https://centos.org/" 2024-04-22T14:58:55.884 INFO:teuthology.orchestra.run.smithi172.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-22T14:58:55.884 INFO:teuthology.orchestra.run.smithi172.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-22T14:58:55.884 INFO:teuthology.orchestra.run.smithi172.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-22T14:58:55.884 INFO:teuthology.lock.ops:Updating smithi172.front.sepia.ceph.com on lock server 2024-04-22T14:58:55.907 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-22T14:58:55.916 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-22T14:58:55.921 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-22T14:58:55.921 DEBUG:teuthology.orchestra.run.smithi145:> test '!' -e /home/ubuntu/cephtest 2024-04-22T14:58:55.924 DEBUG:teuthology.orchestra.run.smithi172:> test '!' -e /home/ubuntu/cephtest 2024-04-22T14:58:55.939 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-22T14:58:55.945 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-22T14:58:55.945 DEBUG:teuthology.orchestra.run.smithi145:> test -z $(ls -A /var/lib/ceph) 2024-04-22T14:58:55.983 DEBUG:teuthology.orchestra.run.smithi172:> test -z $(ls -A /var/lib/ceph) 2024-04-22T14:58:56.108 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-22T14:58:56.156 INFO:teuthology.run_tasks:Running task kernel... 2024-04-22T14:58:56.170 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-22T14:58:56.170 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-22T14:58:56.171 DEBUG:teuthology.orchestra.run.smithi145:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-22T14:58:56.171 DEBUG:teuthology.orchestra.run.smithi172:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-22T14:58:56.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T14:58:56.187 DEBUG:teuthology.orchestra.run.smithi145:> uname -r 2024-04-22T14:58:56.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T14:58:56.189 DEBUG:teuthology.orchestra.run.smithi172:> uname -r 2024-04-22T14:58:56.244 INFO:teuthology.orchestra.run.smithi145.stdout:5.14.0-437.el9.x86_64 2024-04-22T14:58:56.244 INFO:teuthology.task.kernel:Running kernel on smithi145: 5.14.0-437.el9.x86_64 2024-04-22T14:58:56.245 DEBUG:teuthology.orchestra.run.smithi145:> sudo yum install -y kernel 2024-04-22T14:58:56.246 INFO:teuthology.orchestra.run.smithi172.stdout:5.14.0-437.el9.x86_64 2024-04-22T14:58:56.247 INFO:teuthology.task.kernel:Running kernel on smithi172: 5.14.0-437.el9.x86_64 2024-04-22T14:58:56.247 DEBUG:teuthology.orchestra.run.smithi172:> sudo yum install -y kernel 2024-04-22T14:58:58.388 INFO:teuthology.orchestra.run.smithi172.stdout:CentOS Stream 9 - BaseOS 17 MB/s | 8.0 MB 00:00 2024-04-22T14:58:59.099 INFO:teuthology.orchestra.run.smithi145.stdout:CentOS Stream 9 - BaseOS 6.8 MB/s | 8.0 MB 00:01 2024-04-22T14:59:00.550 INFO:teuthology.orchestra.run.smithi172.stdout:CentOS Stream 9 - AppStream 17 MB/s | 19 MB 00:01 2024-04-22T14:59:00.933 INFO:teuthology.orchestra.run.smithi145.stdout:CentOS Stream 9 - AppStream 25 MB/s | 19 MB 00:00 2024-04-22T14:59:05.048 INFO:teuthology.orchestra.run.smithi172.stdout:CentOS Stream 9 - CRB 7.1 MB/s | 6.2 MB 00:00 2024-04-22T14:59:06.740 INFO:teuthology.orchestra.run.smithi172.stdout:CentOS Stream 9 - Extras packages 48 kB/s | 16 kB 00:00 2024-04-22T14:59:07.858 INFO:teuthology.orchestra.run.smithi145.stdout:CentOS Stream 9 - CRB 1.8 MB/s | 6.2 MB 00:03 2024-04-22T14:59:08.469 INFO:teuthology.orchestra.run.smithi172.stdout:Extra Packages for Enterprise Linux 13 MB/s | 21 MB 00:01 2024-04-22T14:59:09.592 INFO:teuthology.orchestra.run.smithi145.stdout:CentOS Stream 9 - Extras packages 50 kB/s | 16 kB 00:00 2024-04-22T14:59:10.702 INFO:teuthology.orchestra.run.smithi145.stdout:Extra Packages for Enterprise Linux 21 MB/s | 21 MB 00:00 2024-04-22T14:59:13.566 INFO:teuthology.orchestra.run.smithi172.stdout:lab-extras 13 kB/s | 1.7 kB 00:00 2024-04-22T14:59:15.430 INFO:teuthology.orchestra.run.smithi172.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-22T14:59:15.430 INFO:teuthology.orchestra.run.smithi172.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-22T14:59:15.526 INFO:teuthology.orchestra.run.smithi172.stdout:Dependencies resolved. 2024-04-22T14:59:15.527 INFO:teuthology.orchestra.run.smithi172.stdout:Nothing to do. 2024-04-22T14:59:15.527 INFO:teuthology.orchestra.run.smithi172.stdout:Complete! 2024-04-22T14:59:15.628 DEBUG:teuthology.orchestra.run.smithi172:> echo no | sudo yum reinstall kernel || true 2024-04-22T14:59:15.718 INFO:teuthology.orchestra.run.smithi145.stdout:lab-extras 36 kB/s | 1.7 kB 00:00 2024-04-22T14:59:16.135 INFO:teuthology.orchestra.run.smithi172.stdout:Last metadata expiration check: 0:00:03 ago on Mon 22 Apr 2024 02:59:13 PM UTC. 2024-04-22T14:59:16.264 INFO:teuthology.orchestra.run.smithi172.stdout:Dependencies resolved. 2024-04-22T14:59:16.273 INFO:teuthology.orchestra.run.smithi172.stdout:================================================================================ 2024-04-22T14:59:16.273 INFO:teuthology.orchestra.run.smithi172.stdout: Package Architecture Version Repository Size 2024-04-22T14:59:16.273 INFO:teuthology.orchestra.run.smithi172.stdout:================================================================================ 2024-04-22T14:59:16.273 INFO:teuthology.orchestra.run.smithi172.stdout:Reinstalling: 2024-04-22T14:59:16.273 INFO:teuthology.orchestra.run.smithi172.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-22T14:59:16.274 INFO:teuthology.orchestra.run.smithi172.stdout: 2024-04-22T14:59:16.274 INFO:teuthology.orchestra.run.smithi172.stdout:Transaction Summary 2024-04-22T14:59:16.274 INFO:teuthology.orchestra.run.smithi172.stdout:================================================================================ 2024-04-22T14:59:16.274 INFO:teuthology.orchestra.run.smithi172.stdout: 2024-04-22T14:59:16.275 INFO:teuthology.orchestra.run.smithi172.stdout:Total download size: 255 k 2024-04-22T14:59:16.275 INFO:teuthology.orchestra.run.smithi172.stdout:Installed size: 0 2024-04-22T14:59:16.275 INFO:teuthology.orchestra.run.smithi172.stderr:Operation aborted. 2024-04-22T14:59:16.338 INFO:teuthology.orchestra.run.smithi172.stdout:Is this ok [y/N]: 2024-04-22T14:59:16.339 DEBUG:teuthology.orchestra.run.smithi172:> sudo yum reinstall -y kernel || true 2024-04-22T14:59:16.840 INFO:teuthology.orchestra.run.smithi172.stdout:Last metadata expiration check: 0:00:03 ago on Mon 22 Apr 2024 02:59:13 PM UTC. 2024-04-22T14:59:16.957 INFO:teuthology.orchestra.run.smithi172.stdout:Dependencies resolved. 2024-04-22T14:59:16.957 INFO:teuthology.orchestra.run.smithi172.stdout:================================================================================ 2024-04-22T14:59:16.957 INFO:teuthology.orchestra.run.smithi172.stdout: Package Architecture Version Repository Size 2024-04-22T14:59:16.957 INFO:teuthology.orchestra.run.smithi172.stdout:================================================================================ 2024-04-22T14:59:16.958 INFO:teuthology.orchestra.run.smithi172.stdout:Reinstalling: 2024-04-22T14:59:16.958 INFO:teuthology.orchestra.run.smithi172.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-22T14:59:16.958 INFO:teuthology.orchestra.run.smithi172.stdout: 2024-04-22T14:59:16.958 INFO:teuthology.orchestra.run.smithi172.stdout:Transaction Summary 2024-04-22T14:59:16.958 INFO:teuthology.orchestra.run.smithi172.stdout:================================================================================ 2024-04-22T14:59:16.958 INFO:teuthology.orchestra.run.smithi172.stdout: 2024-04-22T14:59:16.958 INFO:teuthology.orchestra.run.smithi172.stdout:Total download size: 255 k 2024-04-22T14:59:16.958 INFO:teuthology.orchestra.run.smithi172.stdout:Installed size: 0 2024-04-22T14:59:16.958 INFO:teuthology.orchestra.run.smithi172.stdout:Downloading Packages: 2024-04-22T14:59:17.557 INFO:teuthology.orchestra.run.smithi145.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-22T14:59:17.557 INFO:teuthology.orchestra.run.smithi145.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-22T14:59:17.643 INFO:teuthology.orchestra.run.smithi145.stdout:Dependencies resolved. 2024-04-22T14:59:17.645 INFO:teuthology.orchestra.run.smithi145.stdout:Nothing to do. 2024-04-22T14:59:17.645 INFO:teuthology.orchestra.run.smithi145.stdout:Complete! 2024-04-22T14:59:17.768 INFO:teuthology.orchestra.run.smithi172.stdout:kernel-5.14.0-437.el9.x86_64.rpm 547 kB/s | 255 kB 00:00 2024-04-22T14:59:17.769 INFO:teuthology.orchestra.run.smithi172.stdout:-------------------------------------------------------------------------------- 2024-04-22T14:59:17.769 INFO:teuthology.orchestra.run.smithi172.stdout:Total 314 kB/s | 255 kB 00:00 2024-04-22T14:59:17.775 DEBUG:teuthology.orchestra.run.smithi145:> echo no | sudo yum reinstall kernel || true 2024-04-22T14:59:17.790 INFO:teuthology.orchestra.run.smithi172.stdout:Running transaction check 2024-04-22T14:59:17.981 INFO:teuthology.orchestra.run.smithi172.stdout:Transaction check succeeded. 2024-04-22T14:59:17.981 INFO:teuthology.orchestra.run.smithi172.stdout:Running transaction test 2024-04-22T14:59:17.986 INFO:teuthology.orchestra.run.smithi172.stdout:Transaction test succeeded. 2024-04-22T14:59:17.986 INFO:teuthology.orchestra.run.smithi172.stdout:Running transaction 2024-04-22T14:59:18.142 INFO:teuthology.orchestra.run.smithi172.stdout: Preparing : 1/1 2024-04-22T14:59:18.227 INFO:teuthology.orchestra.run.smithi172.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-22T14:59:18.228 INFO:teuthology.orchestra.run.smithi172.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-22T14:59:18.315 INFO:teuthology.orchestra.run.smithi145.stdout:Last metadata expiration check: 0:00:03 ago on Mon 22 Apr 2024 02:59:15 PM UTC. 2024-04-22T14:59:18.439 INFO:teuthology.orchestra.run.smithi145.stdout:Dependencies resolved. 2024-04-22T14:59:18.456 INFO:teuthology.orchestra.run.smithi145.stdout:================================================================================ 2024-04-22T14:59:18.456 INFO:teuthology.orchestra.run.smithi145.stdout: Package Architecture Version Repository Size 2024-04-22T14:59:18.456 INFO:teuthology.orchestra.run.smithi145.stdout:================================================================================ 2024-04-22T14:59:18.456 INFO:teuthology.orchestra.run.smithi145.stdout:Reinstalling: 2024-04-22T14:59:18.456 INFO:teuthology.orchestra.run.smithi145.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-22T14:59:18.456 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-04-22T14:59:18.456 INFO:teuthology.orchestra.run.smithi145.stdout:Transaction Summary 2024-04-22T14:59:18.456 INFO:teuthology.orchestra.run.smithi145.stdout:================================================================================ 2024-04-22T14:59:18.456 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-04-22T14:59:18.456 INFO:teuthology.orchestra.run.smithi145.stdout:Total download size: 255 k 2024-04-22T14:59:18.456 INFO:teuthology.orchestra.run.smithi145.stdout:Installed size: 0 2024-04-22T14:59:18.457 INFO:teuthology.orchestra.run.smithi145.stderr:Operation aborted. 2024-04-22T14:59:18.522 INFO:teuthology.orchestra.run.smithi145.stdout:Is this ok [y/N]: 2024-04-22T14:59:18.522 DEBUG:teuthology.orchestra.run.smithi145:> sudo yum reinstall -y kernel || true 2024-04-22T14:59:18.956 INFO:teuthology.orchestra.run.smithi172.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-22T14:59:18.956 INFO:teuthology.orchestra.run.smithi172.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-22T14:59:19.021 INFO:teuthology.orchestra.run.smithi145.stdout:Last metadata expiration check: 0:00:04 ago on Mon 22 Apr 2024 02:59:15 PM UTC. 2024-04-22T14:59:19.139 INFO:teuthology.orchestra.run.smithi145.stdout:Dependencies resolved. 2024-04-22T14:59:19.140 INFO:teuthology.orchestra.run.smithi145.stdout:================================================================================ 2024-04-22T14:59:19.140 INFO:teuthology.orchestra.run.smithi145.stdout: Package Architecture Version Repository Size 2024-04-22T14:59:19.140 INFO:teuthology.orchestra.run.smithi145.stdout:================================================================================ 2024-04-22T14:59:19.141 INFO:teuthology.orchestra.run.smithi145.stdout:Reinstalling: 2024-04-22T14:59:19.141 INFO:teuthology.orchestra.run.smithi145.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-22T14:59:19.141 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-04-22T14:59:19.141 INFO:teuthology.orchestra.run.smithi145.stdout:Transaction Summary 2024-04-22T14:59:19.141 INFO:teuthology.orchestra.run.smithi145.stdout:================================================================================ 2024-04-22T14:59:19.141 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-04-22T14:59:19.141 INFO:teuthology.orchestra.run.smithi145.stdout:Total download size: 255 k 2024-04-22T14:59:19.141 INFO:teuthology.orchestra.run.smithi145.stdout:Installed size: 0 2024-04-22T14:59:19.141 INFO:teuthology.orchestra.run.smithi145.stdout:Downloading Packages: 2024-04-22T14:59:19.645 INFO:teuthology.orchestra.run.smithi145.stdout:kernel-5.14.0-437.el9.x86_64.rpm 1.0 MB/s | 255 kB 00:00 2024-04-22T14:59:19.645 INFO:teuthology.orchestra.run.smithi145.stdout:-------------------------------------------------------------------------------- 2024-04-22T14:59:19.645 INFO:teuthology.orchestra.run.smithi145.stdout:Total 505 kB/s | 255 kB 00:00 2024-04-22T14:59:19.661 INFO:teuthology.orchestra.run.smithi172.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-22T14:59:19.662 INFO:teuthology.orchestra.run.smithi172.stdout: 2024-04-22T14:59:19.662 INFO:teuthology.orchestra.run.smithi172.stdout:Reinstalled: 2024-04-22T14:59:19.662 INFO:teuthology.orchestra.run.smithi172.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-22T14:59:19.662 INFO:teuthology.orchestra.run.smithi172.stdout: 2024-04-22T14:59:19.663 INFO:teuthology.orchestra.run.smithi172.stdout:Complete! 2024-04-22T14:59:19.663 INFO:teuthology.orchestra.run.smithi145.stdout:Running transaction check 2024-04-22T14:59:19.832 DEBUG:teuthology.orchestra.run.smithi172:> rpm -q kernel | sort -rV | head -n 1 2024-04-22T14:59:19.862 INFO:teuthology.orchestra.run.smithi145.stdout:Transaction check succeeded. 2024-04-22T14:59:19.862 INFO:teuthology.orchestra.run.smithi145.stdout:Running transaction test 2024-04-22T14:59:19.867 INFO:teuthology.orchestra.run.smithi145.stdout:Transaction test succeeded. 2024-04-22T14:59:19.868 INFO:teuthology.orchestra.run.smithi145.stdout:Running transaction 2024-04-22T14:59:19.879 INFO:teuthology.orchestra.run.smithi172.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-22T14:59:19.879 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-22T14:59:19.879 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-22T14:59:19.880 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-22T14:59:19.880 DEBUG:teuthology.orchestra.run.smithi172:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-22T14:59:19.911 INFO:teuthology.orchestra.run.smithi172.stdout:ttyS1 2024-04-22T14:59:19.931 DEBUG:teuthology.parallel:result is None 2024-04-22T14:59:19.996 INFO:teuthology.orchestra.run.smithi145.stdout: Preparing : 1/1 2024-04-22T14:59:20.087 INFO:teuthology.orchestra.run.smithi145.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-22T14:59:20.088 INFO:teuthology.orchestra.run.smithi145.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-22T14:59:20.761 INFO:teuthology.orchestra.run.smithi145.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-22T14:59:20.761 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-22T14:59:21.469 INFO:teuthology.orchestra.run.smithi145.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-22T14:59:21.469 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-04-22T14:59:21.469 INFO:teuthology.orchestra.run.smithi145.stdout:Reinstalled: 2024-04-22T14:59:21.469 INFO:teuthology.orchestra.run.smithi145.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-22T14:59:21.469 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-04-22T14:59:21.469 INFO:teuthology.orchestra.run.smithi145.stdout:Complete! 2024-04-22T14:59:21.659 DEBUG:teuthology.orchestra.run.smithi145:> rpm -q kernel | sort -rV | head -n 1 2024-04-22T14:59:21.705 INFO:teuthology.orchestra.run.smithi145.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-22T14:59:21.706 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-22T14:59:21.706 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-22T14:59:21.706 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-22T14:59:21.706 DEBUG:teuthology.orchestra.run.smithi145:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-22T14:59:21.736 INFO:teuthology.orchestra.run.smithi145.stdout:ttyS1 2024-04-22T14:59:21.762 DEBUG:teuthology.parallel:result is None 2024-04-22T14:59:21.762 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-22T14:59:21.773 INFO:teuthology.task.internal:Creating test directory... 2024-04-22T14:59:21.773 DEBUG:teuthology.orchestra.run.smithi145:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-22T14:59:21.805 DEBUG:teuthology.orchestra.run.smithi172:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-22T14:59:21.823 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-22T14:59:21.830 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-22T14:59:21.837 INFO:teuthology.task.internal:Creating archive directory... 2024-04-22T14:59:21.837 DEBUG:teuthology.orchestra.run.smithi145:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-22T14:59:21.860 DEBUG:teuthology.orchestra.run.smithi172:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-22T14:59:21.892 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-22T14:59:21.911 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-22T14:59:21.911 DEBUG:teuthology.orchestra.run.smithi145:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-22T14:59:21.922 DEBUG:teuthology.orchestra.run.smithi172:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-22T14:59:21.947 INFO:teuthology.orchestra.run.smithi145.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-22T14:59:21.956 INFO:teuthology.orchestra.run.smithi145.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-22T14:59:21.999 INFO:teuthology.orchestra.run.smithi172.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-22T14:59:22.010 INFO:teuthology.orchestra.run.smithi172.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-22T14:59:22.011 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-22T14:59:22.020 INFO:teuthology.task.internal:Configuring sudo... 2024-04-22T14:59:22.021 DEBUG:teuthology.orchestra.run.smithi145:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-22T14:59:22.023 DEBUG:teuthology.orchestra.run.smithi172:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-22T14:59:22.049 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-22T14:59:22.059 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-22T14:59:22.059 DEBUG:teuthology.orchestra.run.smithi145:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-22T14:59:22.087 DEBUG:teuthology.orchestra.run.smithi172:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-22T14:59:22.105 DEBUG:teuthology.orchestra.run.smithi145:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-22T14:59:22.180 DEBUG:teuthology.orchestra.run.smithi145:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-22T14:59:22.237 DEBUG:teuthology.orchestra.run.smithi145:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-22T14:59:22.319 DEBUG:teuthology.orchestra.run.smithi145:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-22T14:59:22.357 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-04-22T14:59:22.357 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-22T14:59:22.421 DEBUG:teuthology.orchestra.run.smithi172:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-22T14:59:22.465 DEBUG:teuthology.orchestra.run.smithi172:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-22T14:59:22.511 DEBUG:teuthology.orchestra.run.smithi172:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-22T14:59:22.598 DEBUG:teuthology.orchestra.run.smithi172:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-22T14:59:22.639 DEBUG:teuthology.orchestra.run.smithi172:> set -ex 2024-04-22T14:59:22.639 DEBUG:teuthology.orchestra.run.smithi172:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-22T14:59:22.705 DEBUG:teuthology.orchestra.run.smithi145:> sudo service rsyslog restart 2024-04-22T14:59:22.707 DEBUG:teuthology.orchestra.run.smithi172:> sudo service rsyslog restart 2024-04-22T14:59:22.745 INFO:teuthology.orchestra.run.smithi145.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-22T14:59:22.792 INFO:teuthology.orchestra.run.smithi172.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-22T14:59:23.251 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-22T14:59:23.262 INFO:teuthology.task.internal:Starting timer... 2024-04-22T14:59:23.262 INFO:teuthology.run_tasks:Running task pcp... 2024-04-22T14:59:23.274 INFO:teuthology.run_tasks:Running task selinux... 2024-04-22T14:59:23.285 DEBUG:teuthology.orchestra.run.smithi145:> sudo service auditd rotate 2024-04-22T14:59:23.324 INFO:teuthology.orchestra.run.smithi145.stdout:Rotating logs: 2024-04-22T14:59:23.326 DEBUG:teuthology.orchestra.run.smithi172:> sudo service auditd rotate 2024-04-22T14:59:23.393 INFO:teuthology.orchestra.run.smithi172.stdout:Rotating logs: 2024-04-22T14:59:23.395 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-22T14:59:23.395 DEBUG:teuthology.orchestra.run.smithi145:> /usr/sbin/getenforce 2024-04-22T14:59:23.419 INFO:teuthology.orchestra.run.smithi145.stdout:Permissive 2024-04-22T14:59:23.419 DEBUG:teuthology.orchestra.run.smithi172:> /usr/sbin/getenforce 2024-04-22T14:59:23.441 INFO:teuthology.orchestra.run.smithi172.stdout:Permissive 2024-04-22T14:59:23.442 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi145.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi172.front.sepia.ceph.com': 'permissive'} 2024-04-22T14:59:23.442 DEBUG:teuthology.orchestra.run.smithi145:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-22T14:59:23.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T14:59:23.484 DEBUG:teuthology.orchestra.run.smithi172:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-22T14:59:23.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T14:59:23.509 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-22T14:59:23.509 DEBUG:teuthology.orchestra.run.smithi145:> sudo /usr/sbin/setenforce permissive 2024-04-22T14:59:23.548 DEBUG:teuthology.orchestra.run.smithi172:> sudo /usr/sbin/setenforce permissive 2024-04-22T14:59:23.576 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-22T14:59:23.586 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-22T14:59:23.603 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-22T14:59:23.604 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi145.front.sepia.ceph.com,smithi172.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-22T15:03:49.120 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi145.front.sepia.ceph.com'), Remote(name='ubuntu@smithi172.front.sepia.ceph.com')] 2024-04-22T15:03:49.121 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi145.front.sepia.ceph.com' 2024-04-22T15:03:49.126 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi145.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-22T15:03:49.251 DEBUG:teuthology.orchestra.run.smithi145:> true 2024-04-22T15:03:49.334 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi145.front.sepia.ceph.com' 2024-04-22T15:03:49.334 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi172.front.sepia.ceph.com' 2024-04-22T15:03:49.335 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi172.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-22T15:03:49.461 DEBUG:teuthology.orchestra.run.smithi172:> true 2024-04-22T15:03:49.542 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi172.front.sepia.ceph.com' 2024-04-22T15:03:49.542 INFO:teuthology.run_tasks:Running task clock... 2024-04-22T15:03:49.555 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-22T15:03:49.556 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-22T15:03:49.556 DEBUG:teuthology.orchestra.run.smithi145:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-22T15:03:49.559 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-22T15:03:49.560 DEBUG:teuthology.orchestra.run.smithi172:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-22T15:03:49.591 INFO:teuthology.orchestra.run.smithi145.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-22T15:03:49.602 INFO:teuthology.orchestra.run.smithi145.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-22T15:03:49.614 INFO:teuthology.orchestra.run.smithi172.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-22T15:03:49.625 INFO:teuthology.orchestra.run.smithi145.stderr:sudo: ntpd: command not found 2024-04-22T15:03:49.628 INFO:teuthology.orchestra.run.smithi172.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-22T15:03:49.638 INFO:teuthology.orchestra.run.smithi145.stdout:506 Cannot talk to daemon 2024-04-22T15:03:49.656 INFO:teuthology.orchestra.run.smithi172.stderr:sudo: ntpd: command not found 2024-04-22T15:03:49.656 INFO:teuthology.orchestra.run.smithi145.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-22T15:03:49.662 INFO:teuthology.orchestra.run.smithi172.stdout:506 Cannot talk to daemon 2024-04-22T15:03:49.662 INFO:teuthology.orchestra.run.smithi145.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-22T15:03:49.675 INFO:teuthology.orchestra.run.smithi172.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-22T15:03:49.688 INFO:teuthology.orchestra.run.smithi172.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-22T15:03:49.714 INFO:teuthology.orchestra.run.smithi145.stderr:bash: line 1: ntpq: command not found 2024-04-22T15:03:49.721 INFO:teuthology.orchestra.run.smithi145.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-22T15:03:49.721 INFO:teuthology.orchestra.run.smithi145.stdout:=============================================================================== 2024-04-22T15:03:49.721 INFO:teuthology.orchestra.run.smithi145.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T15:03:49.721 INFO:teuthology.orchestra.run.smithi145.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T15:03:49.721 INFO:teuthology.orchestra.run.smithi145.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T15:03:49.721 INFO:teuthology.orchestra.run.smithi145.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T15:03:49.741 INFO:teuthology.orchestra.run.smithi172.stderr:bash: line 1: ntpq: command not found 2024-04-22T15:03:49.744 INFO:teuthology.orchestra.run.smithi172.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-22T15:03:49.744 INFO:teuthology.orchestra.run.smithi172.stdout:=============================================================================== 2024-04-22T15:03:49.744 INFO:teuthology.orchestra.run.smithi172.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T15:03:49.744 INFO:teuthology.orchestra.run.smithi172.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T15:03:49.744 INFO:teuthology.orchestra.run.smithi172.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T15:03:49.745 INFO:teuthology.orchestra.run.smithi172.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T15:03:49.746 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-22T15:03:49.902 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1, 'mgr/cephadm/use_agent': False}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'log-only-match': ['CEPHADM_'], 'sha1': '5883d494d3973b30b3f03e4a3f0e1e06e3c131aa'} 2024-04-22T15:03:49.902 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-04-22T15:03:49.902 INFO:tasks.cephadm:Cluster fsid is 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:03:49.903 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-22T15:03:49.904 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.145', 'mon.c': '[v2:172.21.15.145:3301,v1:172.21.15.145:6790]', 'mon.b': '172.21.15.172'} 2024-04-22T15:03:49.904 INFO:tasks.cephadm:First mon is mon.a on smithi145 2024-04-22T15:03:49.904 INFO:tasks.cephadm:First mgr is y 2024-04-22T15:03:49.904 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-22T15:03:49.904 DEBUG:teuthology.orchestra.run.smithi145:> sudo hostname $(hostname -s) 2024-04-22T15:03:49.929 DEBUG:teuthology.orchestra.run.smithi172:> sudo hostname $(hostname -s) 2024-04-22T15:03:49.955 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-04-22T15:03:49.956 DEBUG:teuthology.orchestra.run.smithi145:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-22T15:03:50.157 INFO:teuthology.orchestra.run.smithi145.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 22 15:03 /home/ubuntu/cephtest/cephadm 2024-04-22T15:03:50.157 DEBUG:teuthology.orchestra.run.smithi172:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-22T15:03:50.462 INFO:teuthology.orchestra.run.smithi172.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 22 15:03 /home/ubuntu/cephtest/cephadm 2024-04-22T15:03:50.462 DEBUG:teuthology.orchestra.run.smithi145:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-22T15:03:50.479 DEBUG:teuthology.orchestra.run.smithi172:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-22T15:03:50.527 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-04-22T15:03:50.528 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-22T15:03:50.535 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-22T15:03:52.050 INFO:teuthology.orchestra.run.smithi172.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-22T15:03:52.077 INFO:teuthology.orchestra.run.smithi145.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-22T15:04:20.775 INFO:teuthology.orchestra.run.smithi172.stdout:{ 2024-04-22T15:04:20.776 INFO:teuthology.orchestra.run.smithi172.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-22T15:04:20.776 INFO:teuthology.orchestra.run.smithi172.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-22T15:04:20.777 INFO:teuthology.orchestra.run.smithi172.stdout: "repo_digests": [ 2024-04-22T15:04:20.777 INFO:teuthology.orchestra.run.smithi172.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-22T15:04:20.777 INFO:teuthology.orchestra.run.smithi172.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-22T15:04:20.777 INFO:teuthology.orchestra.run.smithi172.stdout: ] 2024-04-22T15:04:20.777 INFO:teuthology.orchestra.run.smithi172.stdout:} 2024-04-22T15:04:21.024 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:04:21.024 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-22T15:04:21.024 INFO:teuthology.orchestra.run.smithi145.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-22T15:04:21.024 INFO:teuthology.orchestra.run.smithi145.stdout: "repo_digests": [ 2024-04-22T15:04:21.025 INFO:teuthology.orchestra.run.smithi145.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-22T15:04:21.025 INFO:teuthology.orchestra.run.smithi145.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-22T15:04:21.025 INFO:teuthology.orchestra.run.smithi145.stdout: ] 2024-04-22T15:04:21.025 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:04:21.050 DEBUG:teuthology.orchestra.run.smithi145:> sudo mkdir -p /etc/ceph 2024-04-22T15:04:21.100 DEBUG:teuthology.orchestra.run.smithi172:> sudo mkdir -p /etc/ceph 2024-04-22T15:04:21.127 DEBUG:teuthology.orchestra.run.smithi145:> sudo chmod 777 /etc/ceph 2024-04-22T15:04:21.165 DEBUG:teuthology.orchestra.run.smithi172:> sudo chmod 777 /etc/ceph 2024-04-22T15:04:21.196 INFO:tasks.cephadm:Writing seed config... 2024-04-22T15:04:21.197 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-04-22T15:04:21.197 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-22T15:04:21.197 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-22T15:04:21.197 INFO:tasks.cephadm: override: [mgr] mgr/cephadm/use_agent = False 2024-04-22T15:04:21.197 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-22T15:04:21.197 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-22T15:04:21.197 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-22T15:04:21.198 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-22T15:04:21.198 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-22T15:04:21.198 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-04-22T15:04:21.198 DEBUG:teuthology.orchestra.run.smithi145:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-22T15:04:21.223 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 869afcfa-00b9-11ef-bc93-c7b262605968 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = true bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 mgr/cephadm/use_agent = False [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-04-22T15:04:21.224 DEBUG:teuthology.orchestra.run.smithi145:mon.a> sudo journalctl -f -n 0 -u ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mon.a.service 2024-04-22T15:04:21.267 DEBUG:teuthology.orchestra.run.smithi145:mgr.y> sudo journalctl -f -n 0 -u ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mgr.y.service 2024-04-22T15:04:21.310 INFO:tasks.cephadm:Bootstrapping... 2024-04-22T15:04:21.313 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.145 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-22T15:04:21.505 INFO:teuthology.orchestra.run.smithi145.stderr:-------------------------------------------------------------------------------- 2024-04-22T15:04:21.506 INFO:teuthology.orchestra.run.smithi145.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '869afcfa-00b9-11ef-bc93-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.145', '--skip-admin-label'] 2024-04-22T15:04:21.573 INFO:teuthology.orchestra.run.smithi145.stderr:/bin/podman: 4.9.4-dev 2024-04-22T15:04:21.575 INFO:teuthology.orchestra.run.smithi145.stderr:Verifying podman|docker is present... 2024-04-22T15:04:21.597 INFO:teuthology.orchestra.run.smithi145.stderr:/bin/podman: 4.9.4-dev 2024-04-22T15:04:21.599 INFO:teuthology.orchestra.run.smithi145.stderr:Verifying lvm2 is present... 2024-04-22T15:04:21.600 INFO:teuthology.orchestra.run.smithi145.stderr:Verifying time synchronization is in place... 2024-04-22T15:04:21.606 INFO:teuthology.orchestra.run.smithi145.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-22T15:04:21.612 INFO:teuthology.orchestra.run.smithi145.stderr:systemctl: inactive 2024-04-22T15:04:21.619 INFO:teuthology.orchestra.run.smithi145.stderr:systemctl: enabled 2024-04-22T15:04:21.625 INFO:teuthology.orchestra.run.smithi145.stderr:systemctl: active 2024-04-22T15:04:21.626 INFO:teuthology.orchestra.run.smithi145.stderr:Unit chronyd.service is enabled and running 2024-04-22T15:04:21.626 INFO:teuthology.orchestra.run.smithi145.stderr:Repeating the final host check... 2024-04-22T15:04:21.700 INFO:teuthology.orchestra.run.smithi145.stderr:/bin/podman: 4.9.4-dev 2024-04-22T15:04:21.703 INFO:teuthology.orchestra.run.smithi145.stderr:podman (/bin/podman) version 4.9.4 is present 2024-04-22T15:04:21.703 INFO:teuthology.orchestra.run.smithi145.stderr:systemctl is present 2024-04-22T15:04:21.703 INFO:teuthology.orchestra.run.smithi145.stderr:lvcreate is present 2024-04-22T15:04:21.710 INFO:teuthology.orchestra.run.smithi145.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-22T15:04:21.715 INFO:teuthology.orchestra.run.smithi145.stderr:systemctl: inactive 2024-04-22T15:04:21.722 INFO:teuthology.orchestra.run.smithi145.stderr:systemctl: enabled 2024-04-22T15:04:21.727 INFO:teuthology.orchestra.run.smithi145.stderr:systemctl: active 2024-04-22T15:04:21.728 INFO:teuthology.orchestra.run.smithi145.stderr:Unit chronyd.service is enabled and running 2024-04-22T15:04:21.728 INFO:teuthology.orchestra.run.smithi145.stderr:Host looks OK 2024-04-22T15:04:21.728 INFO:teuthology.orchestra.run.smithi145.stderr:Cluster fsid: 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:04:21.728 INFO:teuthology.orchestra.run.smithi145.stderr:Acquiring lock 140225514520432 on /run/cephadm/869afcfa-00b9-11ef-bc93-c7b262605968.lock 2024-04-22T15:04:21.728 INFO:teuthology.orchestra.run.smithi145.stderr:Lock 140225514520432 acquired on /run/cephadm/869afcfa-00b9-11ef-bc93-c7b262605968.lock 2024-04-22T15:04:21.728 INFO:teuthology.orchestra.run.smithi145.stderr:Verifying IP 172.21.15.145 port 3300 ... 2024-04-22T15:04:21.730 INFO:teuthology.orchestra.run.smithi145.stderr:Verifying IP 172.21.15.145 port 6789 ... 2024-04-22T15:04:21.730 INFO:teuthology.orchestra.run.smithi145.stderr:Base mon IP is 172.21.15.145, final addrv is [v2:172.21.15.145:3300,v1:172.21.15.145:6789] 2024-04-22T15:04:21.731 INFO:teuthology.orchestra.run.smithi145.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.145 metric 100 2024-04-22T15:04:21.732 INFO:teuthology.orchestra.run.smithi145.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.145 metric 100 2024-04-22T15:04:21.734 INFO:teuthology.orchestra.run.smithi145.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-04-22T15:04:21.734 INFO:teuthology.orchestra.run.smithi145.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2024-04-22T15:04:21.734 INFO:teuthology.orchestra.run.smithi145.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2024-04-22T15:04:21.736 INFO:teuthology.orchestra.run.smithi145.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-22T15:04:21.736 INFO:teuthology.orchestra.run.smithi145.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-04-22T15:04:21.737 INFO:teuthology.orchestra.run.smithi145.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-22T15:04:21.737 INFO:teuthology.orchestra.run.smithi145.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-04-22T15:04:21.737 INFO:teuthology.orchestra.run.smithi145.stderr:/sbin/ip: inet6 fe80::8f9:4e9c:12d5:cd3f/64 scope link noprefixroute 2024-04-22T15:04:21.737 INFO:teuthology.orchestra.run.smithi145.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-22T15:04:21.737 INFO:teuthology.orchestra.run.smithi145.stderr:Mon IP `172.21.15.145` is in CIDR network `172.21.0.0/20` 2024-04-22T15:04:21.737 INFO:teuthology.orchestra.run.smithi145.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-22T15:04:21.738 INFO:teuthology.orchestra.run.smithi145.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-22T15:04:21.840 INFO:teuthology.orchestra.run.smithi145.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-04-22T15:04:22.582 INFO:teuthology.orchestra.run.smithi145.stderr:/bin/podman: Getting image source signatures 2024-04-22T15:04:22.582 INFO:teuthology.orchestra.run.smithi145.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-04-22T15:04:22.582 INFO:teuthology.orchestra.run.smithi145.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-04-22T15:04:22.582 INFO:teuthology.orchestra.run.smithi145.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-04-22T15:04:22.582 INFO:teuthology.orchestra.run.smithi145.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-04-22T15:04:22.582 INFO:teuthology.orchestra.run.smithi145.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-04-22T15:04:22.583 INFO:teuthology.orchestra.run.smithi145.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-22T15:04:22.702 INFO:teuthology.orchestra.run.smithi145.stderr:/bin/podman: Writing manifest to image destination 2024-04-22T15:04:22.805 INFO:teuthology.orchestra.run.smithi145.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-22T15:04:23.203 INFO:teuthology.orchestra.run.smithi145.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-22T15:04:23.746 INFO:teuthology.orchestra.run.smithi145.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-22T15:04:23.746 INFO:teuthology.orchestra.run.smithi145.stderr:Extracting ceph user uid/gid from container image... 2024-04-22T15:04:24.112 INFO:teuthology.orchestra.run.smithi145.stderr:stat: 167 167 2024-04-22T15:04:24.726 INFO:teuthology.orchestra.run.smithi145.stderr:Creating initial keys... 2024-04-22T15:04:25.134 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph-authtool: AQB5fCZmK+o7BBAASAjYP2ck0Go94c0o7h65DQ== 2024-04-22T15:04:26.023 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph-authtool: AQB5fCZmB3W3NxAA/mnKSwgagSdEzzMre1U0mw== 2024-04-22T15:04:27.020 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph-authtool: AQB6fCZmZ/iXNhAAwIHj7rFt47dnT+dfDjQDYg== 2024-04-22T15:04:29.122 INFO:teuthology.orchestra.run.smithi145.stderr:Creating initial monmap... 2024-04-22T15:04:31.015 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-22T15:04:31.016 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-04-22T15:04:31.016 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:04:31.016 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-22T15:04:31.521 INFO:teuthology.orchestra.run.smithi145.stderr:monmaptool for a [v2:172.21.15.145:3300,v1:172.21.15.145:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-22T15:04:31.521 INFO:teuthology.orchestra.run.smithi145.stderr:setting min_mon_release = octopus 2024-04-22T15:04:31.521 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/monmaptool: set fsid to 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:04:31.521 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-22T15:04:31.521 INFO:teuthology.orchestra.run.smithi145.stderr: 2024-04-22T15:04:31.521 INFO:teuthology.orchestra.run.smithi145.stderr:Creating mon... 2024-04-22T15:04:32.601 INFO:teuthology.orchestra.run.smithi145.stderr:create mon.a on 2024-04-22T15:04:32.800 INFO:teuthology.orchestra.run.smithi145.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-22T15:04:32.945 INFO:teuthology.orchestra.run.smithi145.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-869afcfa-00b9-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-869afcfa-00b9-11ef-bc93-c7b262605968.target. 2024-04-22T15:04:32.945 INFO:teuthology.orchestra.run.smithi145.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-869afcfa-00b9-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-869afcfa-00b9-11ef-bc93-c7b262605968.target. 2024-04-22T15:04:33.251 INFO:teuthology.orchestra.run.smithi145.stderr:systemctl: Failed to reset failed state of unit ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mon.a.service: Unit ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mon.a.service not loaded. 2024-04-22T15:04:33.259 INFO:teuthology.orchestra.run.smithi145.stderr:systemctl: Created symlink /etc/systemd/system/ceph-869afcfa-00b9-11ef-bc93-c7b262605968.target.wants/ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mon.a.service → /etc/systemd/system/ceph-869afcfa-00b9-11ef-bc93-c7b262605968@.service. 2024-04-22T15:04:33.949 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:33 smithi145 podman[21535]: 2024-04-22 15:04:33.903776312 +0000 UTC m=+0.343786797 container start 59b62be89b2af7841ac872e9a79558c829322088e831f7351430ac186685fbdc (image=quay.io/ceph/ceph:v17.2.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-a, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, vcs-type=git, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, distribution-scope=public, io.openshift.expose-services=, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux ) 2024-04-22T15:04:34.002 INFO:teuthology.orchestra.run.smithi145.stderr:systemctl: disabled 2024-04-22T15:04:34.010 INFO:teuthology.orchestra.run.smithi145.stderr:systemctl: inactive 2024-04-22T15:04:34.010 INFO:teuthology.orchestra.run.smithi145.stderr:firewalld.service is not enabled 2024-04-22T15:04:34.010 INFO:teuthology.orchestra.run.smithi145.stderr:Not possible to enable service . firewalld.service is not available 2024-04-22T15:04:34.011 INFO:teuthology.orchestra.run.smithi145.stderr:Waiting for mon to start... 2024-04-22T15:04:34.011 INFO:teuthology.orchestra.run.smithi145.stderr:Waiting for mon... 2024-04-22T15:04:34.235 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:33 smithi145 ceph-mon[21555]: mkfs 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:04:34.235 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:33 smithi145 ceph-mon[21555]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-22T15:04:34.235 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:33 smithi145 bash[21535]: 59b62be89b2af7841ac872e9a79558c829322088e831f7351430ac186685fbdc 2024-04-22T15:04:34.235 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:33 smithi145 systemd[1]: Started Ceph mon.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:04:34.528 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: cluster: 2024-04-22T15:04:34.529 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: id: 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:04:34.529 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-04-22T15:04:34.529 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: 2024-04-22T15:04:34.529 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: services: 2024-04-22T15:04:34.529 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.582651s) 2024-04-22T15:04:34.529 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: mgr: no daemons active 2024-04-22T15:04:34.529 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-04-22T15:04:34.529 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: 2024-04-22T15:04:34.529 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: data: 2024-04-22T15:04:34.529 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-04-22T15:04:34.529 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-04-22T15:04:34.529 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-04-22T15:04:34.529 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: pgs: 2024-04-22T15:04:34.530 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: 2024-04-22T15:04:34.954 INFO:teuthology.orchestra.run.smithi145.stderr:mon is available 2024-04-22T15:04:34.955 INFO:teuthology.orchestra.run.smithi145.stderr:Assimilating anything we can from ceph.conf... 2024-04-22T15:04:35.212 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:34 smithi145 ceph-mon[21555]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-22T15:04:35.213 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:34 smithi145 ceph-mon[21555]: monmap e1: 1 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0]} 2024-04-22T15:04:35.213 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:34 smithi145 ceph-mon[21555]: fsmap 2024-04-22T15:04:35.213 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:34 smithi145 ceph-mon[21555]: osdmap e1: 0 total, 0 up, 0 in 2024-04-22T15:04:35.213 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:34 smithi145 ceph-mon[21555]: mgrmap e1: no daemons active 2024-04-22T15:04:35.213 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:34 smithi145 ceph-mon[21555]: from='client.? 172.21.15.145:0/2958196393' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T15:04:35.467 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: 2024-04-22T15:04:35.467 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: [global] 2024-04-22T15:04:35.467 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: fsid = 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:04:35.467 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.145:3300,v1:172.21.15.145:6789] 2024-04-22T15:04:35.467 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-22T15:04:35.467 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-22T15:04:35.467 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-22T15:04:35.467 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-22T15:04:35.468 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: 2024-04-22T15:04:35.468 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: [mgr] 2024-04-22T15:04:35.468 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: mgr/cephadm/use_agent = False 2024-04-22T15:04:35.468 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-22T15:04:35.468 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: 2024-04-22T15:04:35.468 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: [osd] 2024-04-22T15:04:35.468 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-22T15:04:35.468 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-22T15:04:35.985 INFO:teuthology.orchestra.run.smithi145.stderr:Generating new minimal ceph.conf... 2024-04-22T15:04:36.084 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:35 smithi145 ceph-mon[21555]: from='client.? 172.21.15.145:0/1086624769' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-22T15:04:36.084 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:35 smithi145 ceph-mon[21555]: from='client.? 172.21.15.145:0/1086624769' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-22T15:04:37.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:36 smithi145 ceph-mon[21555]: from='client.? 172.21.15.145:0/2652657102' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:04:37.418 INFO:teuthology.orchestra.run.smithi145.stderr:Restarting the monitor... 2024-04-22T15:04:37.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:37 smithi145 systemd[1]: Stopping Ceph mon.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:04:37.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:37 smithi145 bash[21775]: Error: no container with name or ID "ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon.a" found: no such container 2024-04-22T15:04:37.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-a[21551]: 2024-04-22T15:04:37.555+0000 7f79f9400700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:04:37.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-a[21551]: 2024-04-22T15:04:37.555+0000 7f79f9400700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-04-22T15:04:38.076 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:37 smithi145 podman[21786]: 2024-04-22 15:04:37.811700209 +0000 UTC m=+0.341830827 container died 59b62be89b2af7841ac872e9a79558c829322088e831f7351430ac186685fbdc (image=quay.io/ceph/ceph:v17.2.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-a, com.redhat.component=centos-stream-container, vcs-type=git, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , name=centos-stream, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., RELEASE=HEAD, ceph=True, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0) 2024-04-22T15:04:38.328 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 podman[21786]: 2024-04-22 15:04:38.147292354 +0000 UTC m=+0.677422969 container cleanup 59b62be89b2af7841ac872e9a79558c829322088e831f7351430ac186685fbdc (image=quay.io/ceph/ceph:v17.2.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-a, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, architecture=x86_64, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, version=8, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, ceph=True, GIT_BRANCH=HEAD) 2024-04-22T15:04:38.328 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 bash[21786]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-a 2024-04-22T15:04:38.580 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 podman[21801]: 2024-04-22 15:04:38.325325363 +0000 UTC m=+0.509411873 container remove 59b62be89b2af7841ac872e9a79558c829322088e831f7351430ac186685fbdc (image=quay.io/ceph/ceph:v17.2.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-a, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, architecture=x86_64, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, version=8, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, release=754) 2024-04-22T15:04:38.580 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 bash[21814]: Error: no container with name or ID "ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon.a" found: no such container 2024-04-22T15:04:38.580 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mon.a.service: Deactivated successfully. 2024-04-22T15:04:38.580 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 systemd[1]: Stopped Ceph mon.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:04:38.580 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 systemd[1]: Starting Ceph mon.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:04:38.580 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 podman[21880]: 2024-04-22 15:04:38.559422765 +0000 UTC m=+0.098331873 container create 28106a7bf16eac08b626cb72a9b68da00df5d6be5a9f3071115b05e124b62ce6 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-a, RELEASE=HEAD, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., name=centos-stream, version=8, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container) 2024-04-22T15:04:38.581 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 podman[21880]: 2024-04-22 15:04:38.480072116 +0000 UTC m=+0.018981223 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-04-22T15:04:38.893 INFO:teuthology.orchestra.run.smithi145.stderr:Setting mon public_network to 172.21.0.0/20 2024-04-22T15:04:39.101 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 podman[21880]: 2024-04-22 15:04:38.828460184 +0000 UTC m=+0.367369291 container init 28106a7bf16eac08b626cb72a9b68da00df5d6be5a9f3071115b05e124b62ce6 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-a, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., io.buildah.version=1.19.8, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, name=centos-stream, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, distribution-scope=public, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-22T15:04:39.101 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 podman[21880]: 2024-04-22 15:04:38.831896666 +0000 UTC m=+0.370805774 container start 28106a7bf16eac08b626cb72a9b68da00df5d6be5a9f3071115b05e124b62ce6 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-a, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, io.buildah.version=1.19.8, GIT_CLEAN=True, vcs-type=git, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., architecture=x86_64, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=) 2024-04-22T15:04:39.101 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: set uid:gid to 167:167 (ceph:ceph) 2024-04-22T15:04:39.101 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-22T15:04:39.101 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: pidfile_write: ignore empty --pid-file 2024-04-22T15:04:39.101 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: load: jerasure load: lrc 2024-04-22T15:04:39.101 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: RocksDB version: 6.15.5 2024-04-22T15:04:39.101 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-22T15:04:39.101 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Compile date Apr 18 2022 2024-04-22T15:04:39.101 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: DB SUMMARY 2024-04-22T15:04:39.101 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: DB Session ID: YIBTA8RL1BJOBGJQ0K8M 2024-04-22T15:04:39.101 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: CURRENT file: CURRENT 2024-04-22T15:04:39.102 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: IDENTITY file: IDENTITY 2024-04-22T15:04:39.102 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-04-22T15:04:39.102 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-04-22T15:04:39.102 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 82663 ; 2024-04-22T15:04:39.102 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.error_if_exists: 0 2024-04-22T15:04:39.102 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.create_if_missing: 0 2024-04-22T15:04:39.102 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.paranoid_checks: 1 2024-04-22T15:04:39.102 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-22T15:04:39.102 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.env: 0x55e9f22c6860 2024-04-22T15:04:39.102 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.fs: Posix File System 2024-04-22T15:04:39.102 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.info_log: 0x55e9f395bdc0 2024-04-22T15:04:39.102 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.max_file_opening_threads: 16 2024-04-22T15:04:39.102 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.statistics: (nil) 2024-04-22T15:04:39.102 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.use_fsync: 0 2024-04-22T15:04:39.103 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.max_log_file_size: 0 2024-04-22T15:04:39.103 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-22T15:04:39.103 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-22T15:04:39.103 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.keep_log_file_num: 1000 2024-04-22T15:04:39.103 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.recycle_log_file_num: 0 2024-04-22T15:04:39.103 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.allow_fallocate: 1 2024-04-22T15:04:39.103 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.allow_mmap_reads: 0 2024-04-22T15:04:39.103 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.allow_mmap_writes: 0 2024-04-22T15:04:39.103 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.use_direct_reads: 0 2024-04-22T15:04:39.103 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-22T15:04:39.103 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.create_missing_column_families: 0 2024-04-22T15:04:39.103 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.db_log_dir: 2024-04-22T15:04:39.103 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-04-22T15:04:39.104 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-22T15:04:39.104 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-22T15:04:39.104 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-22T15:04:39.104 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-22T15:04:39.104 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-22T15:04:39.104 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-22T15:04:39.104 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.advise_random_on_open: 1 2024-04-22T15:04:39.105 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.db_write_buffer_size: 0 2024-04-22T15:04:39.105 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.write_buffer_manager: 0x55e9f3a4c240 2024-04-22T15:04:39.106 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-22T15:04:39.106 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-22T15:04:39.106 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-22T15:04:39.106 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-22T15:04:39.106 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.rate_limiter: (nil) 2024-04-22T15:04:39.106 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-22T15:04:39.106 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.wal_recovery_mode: 2 2024-04-22T15:04:39.106 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.enable_thread_tracking: 0 2024-04-22T15:04:39.106 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.enable_pipelined_write: 0 2024-04-22T15:04:39.106 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.unordered_write: 0 2024-04-22T15:04:39.106 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-22T15:04:39.106 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-22T15:04:39.106 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-22T15:04:39.106 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-22T15:04:39.107 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.row_cache: None 2024-04-22T15:04:39.107 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.wal_filter: None 2024-04-22T15:04:39.107 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-22T15:04:39.107 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.allow_ingest_behind: 0 2024-04-22T15:04:39.107 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.preserve_deletes: 0 2024-04-22T15:04:39.107 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.two_write_queues: 0 2024-04-22T15:04:39.107 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.manual_wal_flush: 0 2024-04-22T15:04:39.107 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.atomic_flush: 0 2024-04-22T15:04:39.107 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-22T15:04:39.107 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-22T15:04:39.107 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-22T15:04:39.107 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.log_readahead_size: 0 2024-04-22T15:04:39.107 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-22T15:04:39.107 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.best_efforts_recovery: 0 2024-04-22T15:04:39.107 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-22T15:04:39.108 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-22T15:04:39.108 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.allow_data_in_errors: 0 2024-04-22T15:04:39.108 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.db_host_id: __hostname__ 2024-04-22T15:04:39.108 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.max_background_jobs: 2 2024-04-22T15:04:39.108 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.max_background_compactions: -1 2024-04-22T15:04:39.108 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.max_subcompactions: 1 2024-04-22T15:04:39.108 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-22T15:04:39.108 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-22T15:04:39.108 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-22T15:04:39.108 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.max_total_wal_size: 0 2024-04-22T15:04:39.108 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-22T15:04:39.108 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-22T15:04:39.108 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-22T15:04:39.108 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-22T15:04:39.109 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.max_open_files: -1 2024-04-22T15:04:39.109 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.bytes_per_sync: 0 2024-04-22T15:04:39.109 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-22T15:04:39.109 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-22T15:04:39.109 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.compaction_readahead_size: 0 2024-04-22T15:04:39.109 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.max_background_flushes: -1 2024-04-22T15:04:39.109 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Compression algorithms supported: 2024-04-22T15:04:39.109 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-22T15:04:39.109 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: kZSTD supported: 0 2024-04-22T15:04:39.109 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: kXpressCompression supported: 0 2024-04-22T15:04:39.109 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: kLZ4HCCompression supported: 1 2024-04-22T15:04:39.109 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: kLZ4Compression supported: 1 2024-04-22T15:04:39.109 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: kBZip2Compression supported: 0 2024-04-22T15:04:39.109 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: kZlibCompression supported: 1 2024-04-22T15:04:39.110 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: kSnappyCompression supported: 1 2024-04-22T15:04:39.110 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-22T15:04:39.110 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-04-22T15:04:39.110 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-22T15:04:39.110 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-22T15:04:39.110 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.merge_operator: 2024-04-22T15:04:39.110 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.compaction_filter: None 2024-04-22T15:04:39.110 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.compaction_filter_factory: None 2024-04-22T15:04:39.110 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.sst_partitioner_factory: None 2024-04-22T15:04:39.110 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-22T15:04:39.110 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-22T15:04:39.110 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55e9f3927d00) 2024-04-22T15:04:39.110 INFO:journalctl@ceph.mon.a.smithi145.stdout: cache_index_and_filter_blocks: 1 2024-04-22T15:04:39.110 INFO:journalctl@ceph.mon.a.smithi145.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-22T15:04:39.111 INFO:journalctl@ceph.mon.a.smithi145.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-22T15:04:39.111 INFO:journalctl@ceph.mon.a.smithi145.stdout: pin_top_level_index_and_filter: 1 2024-04-22T15:04:39.111 INFO:journalctl@ceph.mon.a.smithi145.stdout: index_type: 0 2024-04-22T15:04:39.111 INFO:journalctl@ceph.mon.a.smithi145.stdout: data_block_index_type: 0 2024-04-22T15:04:39.111 INFO:journalctl@ceph.mon.a.smithi145.stdout: index_shortening: 1 2024-04-22T15:04:39.111 INFO:journalctl@ceph.mon.a.smithi145.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-22T15:04:39.111 INFO:journalctl@ceph.mon.a.smithi145.stdout: hash_index_allow_collision: 1 2024-04-22T15:04:39.111 INFO:journalctl@ceph.mon.a.smithi145.stdout: checksum: 1 2024-04-22T15:04:39.111 INFO:journalctl@ceph.mon.a.smithi145.stdout: no_block_cache: 0 2024-04-22T15:04:39.111 INFO:journalctl@ceph.mon.a.smithi145.stdout: block_cache: 0x55e9f3992170 2024-04-22T15:04:39.111 INFO:journalctl@ceph.mon.a.smithi145.stdout: block_cache_name: BinnedLRUCache 2024-04-22T15:04:39.111 INFO:journalctl@ceph.mon.a.smithi145.stdout: block_cache_options: 2024-04-22T15:04:39.111 INFO:journalctl@ceph.mon.a.smithi145.stdout: capacity : 536870912 2024-04-22T15:04:39.111 INFO:journalctl@ceph.mon.a.smithi145.stdout: num_shard_bits : 4 2024-04-22T15:04:39.111 INFO:journalctl@ceph.mon.a.smithi145.stdout: strict_capacity_limit : 0 2024-04-22T15:04:39.112 INFO:journalctl@ceph.mon.a.smithi145.stdout: high_pri_pool_ratio: 0.000 2024-04-22T15:04:39.112 INFO:journalctl@ceph.mon.a.smithi145.stdout: block_cache_compressed: (nil) 2024-04-22T15:04:39.112 INFO:journalctl@ceph.mon.a.smithi145.stdout: persistent_cache: (nil) 2024-04-22T15:04:39.112 INFO:journalctl@ceph.mon.a.smithi145.stdout: block_size: 4096 2024-04-22T15:04:39.112 INFO:journalctl@ceph.mon.a.smithi145.stdout: block_size_deviation: 10 2024-04-22T15:04:39.112 INFO:journalctl@ceph.mon.a.smithi145.stdout: block_restart_interval: 16 2024-04-22T15:04:39.112 INFO:journalctl@ceph.mon.a.smithi145.stdout: index_block_restart_interval: 1 2024-04-22T15:04:39.112 INFO:journalctl@ceph.mon.a.smithi145.stdout: metadata_block_size: 4096 2024-04-22T15:04:39.112 INFO:journalctl@ceph.mon.a.smithi145.stdout: partition_filters: 0 2024-04-22T15:04:39.112 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_delta_encoding: 1 2024-04-22T15:04:39.112 INFO:journalctl@ceph.mon.a.smithi145.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-22T15:04:39.112 INFO:journalctl@ceph.mon.a.smithi145.stdout: whole_key_filtering: 1 2024-04-22T15:04:39.112 INFO:journalctl@ceph.mon.a.smithi145.stdout: verify_compression: 0 2024-04-22T15:04:39.113 INFO:journalctl@ceph.mon.a.smithi145.stdout: read_amp_bytes_per_bit: 0 2024-04-22T15:04:39.113 INFO:journalctl@ceph.mon.a.smithi145.stdout: format_version: 4 2024-04-22T15:04:39.113 INFO:journalctl@ceph.mon.a.smithi145.stdout: enable_index_compression: 1 2024-04-22T15:04:39.113 INFO:journalctl@ceph.mon.a.smithi145.stdout: block_align: 0 2024-04-22T15:04:39.113 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.write_buffer_size: 33554432 2024-04-22T15:04:39.113 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.max_write_buffer_number: 2 2024-04-22T15:04:39.113 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.compression: NoCompression 2024-04-22T15:04:39.113 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.bottommost_compression: Disabled 2024-04-22T15:04:39.113 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.prefix_extractor: nullptr 2024-04-22T15:04:39.113 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-22T15:04:39.113 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.num_levels: 7 2024-04-22T15:04:39.113 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-22T15:04:39.113 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-22T15:04:39.114 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-22T15:04:39.114 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-22T15:04:39.114 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-22T15:04:39.114 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-22T15:04:39.114 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-22T15:04:39.114 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-22T15:04:39.114 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-22T15:04:39.114 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-22T15:04:39.114 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-22T15:04:39.114 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.compression_opts.level: 32767 2024-04-22T15:04:39.114 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.compression_opts.strategy: 0 2024-04-22T15:04:39.114 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-22T15:04:39.114 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-22T15:04:39.114 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-22T15:04:39.114 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.compression_opts.enabled: false 2024-04-22T15:04:39.115 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-22T15:04:39.115 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-22T15:04:39.115 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-22T15:04:39.115 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.target_file_size_base: 67108864 2024-04-22T15:04:39.115 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-22T15:04:39.115 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-22T15:04:39.115 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-22T15:04:39.115 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-22T15:04:39.115 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-22T15:04:39.115 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-22T15:04:39.115 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-22T15:04:39.115 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-22T15:04:39.115 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-22T15:04:39.115 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-22T15:04:39.116 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-22T15:04:39.116 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-22T15:04:39.116 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-22T15:04:39.116 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.arena_block_size: 4194304 2024-04-22T15:04:39.116 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-22T15:04:39.116 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-22T15:04:39.116 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-22T15:04:39.116 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.disable_auto_compactions: 0 2024-04-22T15:04:39.116 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-22T15:04:39.116 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-22T15:04:39.116 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-22T15:04:39.116 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-22T15:04:39.116 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-22T15:04:39.117 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-22T15:04:39.117 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-22T15:04:39.117 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-22T15:04:39.117 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-22T15:04:39.117 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-22T15:04:39.117 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.table_properties_collectors: 2024-04-22T15:04:39.117 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.inplace_update_support: 0 2024-04-22T15:04:39.117 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-22T15:04:39.117 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-22T15:04:39.117 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-22T15:04:39.117 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-22T15:04:39.117 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.bloom_locality: 0 2024-04-22T15:04:39.117 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.max_successive_merges: 0 2024-04-22T15:04:39.117 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-22T15:04:39.118 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.paranoid_file_checks: 0 2024-04-22T15:04:39.118 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.force_consistency_checks: 1 2024-04-22T15:04:39.118 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.report_bg_io_stats: 0 2024-04-22T15:04:39.118 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.ttl: 2592000 2024-04-22T15:04:39.118 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-22T15:04:39.118 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.enable_blob_files: false 2024-04-22T15:04:39.118 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.min_blob_size: 0 2024-04-22T15:04:39.118 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.blob_file_size: 268435456 2024-04-22T15:04:39.118 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-22T15:04:39.118 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-22T15:04:39.118 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-22T15:04:39.118 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-22T15:04:39.118 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-04-22T15:04:39.119 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-04-22T15:04:39.119 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713798278857274, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-04-22T15:04:39.119 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-04-22T15:04:39.119 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-22T15:04:39.119 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713798278857955, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79213, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77476, "index_size": 225, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10027, "raw_average_key_size": 47, "raw_value_size": 71785, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 211, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1713798278, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "ee7a0575-f010-4b3a-8c13-6d12ef8c1f92", "db_session_id": "YIBTA8RL1BJOBGJQ0K8M"}} 2024-04-22T15:04:39.119 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-04-22T15:04:39.119 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713798278858322, "job": 1, "event": "recovery_finished"} 2024-04-22T15:04:39.119 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-22T15:04:39.119 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55e9f3978a80 2024-04-22T15:04:39.119 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: DB pointer 0x55e9f3988000 2024-04-22T15:04:39.120 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-22T15:04:39.120 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-22T15:04:39.120 INFO:journalctl@ceph.mon.a.smithi145.stdout: ** DB Stats ** 2024-04-22T15:04:39.121 INFO:journalctl@ceph.mon.a.smithi145.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T15:04:39.121 INFO:journalctl@ceph.mon.a.smithi145.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-22T15:04:39.121 INFO:journalctl@ceph.mon.a.smithi145.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-22T15:04:39.121 INFO:journalctl@ceph.mon.a.smithi145.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T15:04:39.121 INFO:journalctl@ceph.mon.a.smithi145.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-22T15:04:39.121 INFO:journalctl@ceph.mon.a.smithi145.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-22T15:04:39.121 INFO:journalctl@ceph.mon.a.smithi145.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T15:04:39.121 INFO:journalctl@ceph.mon.a.smithi145.stdout: 2024-04-22T15:04:39.121 INFO:journalctl@ceph.mon.a.smithi145.stdout: ** Compaction Stats [default] ** 2024-04-22T15:04:39.121 INFO:journalctl@ceph.mon.a.smithi145.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T15:04:39.121 INFO:journalctl@ceph.mon.a.smithi145.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T15:04:39.121 INFO:journalctl@ceph.mon.a.smithi145.stdout: L0 2/0 79.12 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 164.2 0.00 0.00 1 0.000 0 0 2024-04-22T15:04:39.121 INFO:journalctl@ceph.mon.a.smithi145.stdout: Sum 2/0 79.12 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 164.2 0.00 0.00 1 0.000 0 0 2024-04-22T15:04:39.122 INFO:journalctl@ceph.mon.a.smithi145.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 164.2 0.00 0.00 1 0.000 0 0 2024-04-22T15:04:39.122 INFO:journalctl@ceph.mon.a.smithi145.stdout: 2024-04-22T15:04:39.122 INFO:journalctl@ceph.mon.a.smithi145.stdout: ** Compaction Stats [default] ** 2024-04-22T15:04:39.122 INFO:journalctl@ceph.mon.a.smithi145.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T15:04:39.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T15:04:39.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 164.2 0.00 0.00 1 0.000 0 0 2024-04-22T15:04:39.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T15:04:39.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-22T15:04:39.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-22T15:04:39.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-22T15:04:39.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-22T15:04:39.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-22T15:04:39.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: Cumulative compaction: 0.00 GB write, 25.94 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T15:04:39.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: Interval compaction: 0.00 GB write, 25.94 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T15:04:39.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-22T15:04:39.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: 2024-04-22T15:04:39.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-22T15:04:39.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: 2024-04-22T15:04:39.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: ** Compaction Stats [default] ** 2024-04-22T15:04:39.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T15:04:39.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T15:04:39.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: L0 2/0 79.12 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 164.2 0.00 0.00 1 0.000 0 0 2024-04-22T15:04:39.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: Sum 2/0 79.12 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 164.2 0.00 0.00 1 0.000 0 0 2024-04-22T15:04:39.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-22T15:04:39.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: 2024-04-22T15:04:39.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: ** Compaction Stats [default] ** 2024-04-22T15:04:39.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T15:04:39.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T15:04:39.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 164.2 0.00 0.00 1 0.000 0 0 2024-04-22T15:04:39.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T15:04:39.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-22T15:04:39.127 INFO:journalctl@ceph.mon.a.smithi145.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-22T15:04:39.127 INFO:journalctl@ceph.mon.a.smithi145.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-22T15:04:39.127 INFO:journalctl@ceph.mon.a.smithi145.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-22T15:04:39.127 INFO:journalctl@ceph.mon.a.smithi145.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-22T15:04:39.127 INFO:journalctl@ceph.mon.a.smithi145.stdout: Cumulative compaction: 0.00 GB write, 25.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T15:04:39.127 INFO:journalctl@ceph.mon.a.smithi145.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T15:04:39.128 INFO:journalctl@ceph.mon.a.smithi145.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-22T15:04:39.128 INFO:journalctl@ceph.mon.a.smithi145.stdout: 2024-04-22T15:04:39.128 INFO:journalctl@ceph.mon.a.smithi145.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-22T15:04:39.128 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: starting mon.a rank 0 at public addrs [v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0] at bind addrs [v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:04:39.128 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: mon.a@-1(???) e1 preinit fsid 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:04:39.128 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: mon.a@-1(???).mds e1 new map 2024-04-22T15:04:39.129 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: mon.a@-1(???).mds e1 print_map 2024-04-22T15:04:39.129 INFO:journalctl@ceph.mon.a.smithi145.stdout: e1 2024-04-22T15:04:39.129 INFO:journalctl@ceph.mon.a.smithi145.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-22T15:04:39.129 INFO:journalctl@ceph.mon.a.smithi145.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-22T15:04:39.129 INFO:journalctl@ceph.mon.a.smithi145.stdout: legacy client fscid: -1 2024-04-22T15:04:39.129 INFO:journalctl@ceph.mon.a.smithi145.stdout: 2024-04-22T15:04:39.129 INFO:journalctl@ceph.mon.a.smithi145.stdout: No filesystems configured 2024-04-22T15:04:39.129 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-22T15:04:39.129 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T15:04:39.129 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T15:04:39.129 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T15:04:39.129 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-22T15:04:39.129 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: expand_channel_meta expand map: {default=false} 2024-04-22T15:04:39.129 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: expand_channel_meta from 'false' to 'false' 2024-04-22T15:04:39.130 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: expand_channel_meta expanded map: {default=false} 2024-04-22T15:04:39.130 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: expand_channel_meta expand map: {default=info} 2024-04-22T15:04:39.130 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: expand_channel_meta from 'info' to 'info' 2024-04-22T15:04:39.130 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: expand_channel_meta expanded map: {default=info} 2024-04-22T15:04:39.130 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: expand_channel_meta expand map: {default=daemon} 2024-04-22T15:04:39.130 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: expand_channel_meta from 'daemon' to 'daemon' 2024-04-22T15:04:39.130 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: expand_channel_meta expanded map: {default=daemon} 2024-04-22T15:04:39.130 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: expand_channel_meta expand map: {default=debug} 2024-04-22T15:04:39.130 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: expand_channel_meta from 'debug' to 'debug' 2024-04-22T15:04:39.130 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: expand_channel_meta expanded map: {default=debug} 2024-04-22T15:04:39.131 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 bash[21880]: 28106a7bf16eac08b626cb72a9b68da00df5d6be5a9f3071115b05e124b62ce6 2024-04-22T15:04:39.131 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 systemd[1]: Started Ceph mon.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:04:39.131 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-22T15:04:39.131 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: monmap e1: 1 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0]} 2024-04-22T15:04:39.131 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: fsmap 2024-04-22T15:04:39.131 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: osdmap e1: 0 total, 0 up, 0 in 2024-04-22T15:04:39.131 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:38 smithi145 ceph-mon[21899]: mgrmap e1: no daemons active 2024-04-22T15:04:39.974 INFO:teuthology.orchestra.run.smithi145.stderr:Wrote config to /etc/ceph/ceph.conf 2024-04-22T15:04:39.975 INFO:teuthology.orchestra.run.smithi145.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-22T15:04:39.975 INFO:teuthology.orchestra.run.smithi145.stderr:Creating mgr... 2024-04-22T15:04:39.975 INFO:teuthology.orchestra.run.smithi145.stderr:Verifying port 9283 ... 2024-04-22T15:04:40.151 INFO:teuthology.orchestra.run.smithi145.stderr:systemctl: Failed to reset failed state of unit ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mgr.y.service: Unit ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mgr.y.service not loaded. 2024-04-22T15:04:40.159 INFO:teuthology.orchestra.run.smithi145.stderr:systemctl: Created symlink /etc/systemd/system/ceph-869afcfa-00b9-11ef-bc93-c7b262605968.target.wants/ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-869afcfa-00b9-11ef-bc93-c7b262605968@.service. 2024-04-22T15:04:40.311 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:40 smithi145 systemd[1]: Starting Ceph mgr.y for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:04:40.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:40 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2045584094' entity='client.admin' 2024-04-22T15:04:40.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:40 smithi145 podman[22116]: 2024-04-22 15:04:40.426200632 +0000 UTC m=+0.017671900 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-04-22T15:04:40.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:40 smithi145 podman[22116]: 2024-04-22 15:04:40.536414211 +0000 UTC m=+0.127885463 container create 95f50f021d87a9a0c572d17999f705fee933d17dd3d2d43d7e610ea7f4c79dbe (image=quay.io/ceph/ceph:v17.2.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, RELEASE=HEAD, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., release=754, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-22T15:04:40.840 INFO:teuthology.orchestra.run.smithi145.stderr:systemctl: disabled 2024-04-22T15:04:40.851 INFO:teuthology.orchestra.run.smithi145.stderr:systemctl: inactive 2024-04-22T15:04:40.852 INFO:teuthology.orchestra.run.smithi145.stderr:firewalld.service is not enabled 2024-04-22T15:04:40.852 INFO:teuthology.orchestra.run.smithi145.stderr:Not possible to enable service . firewalld.service is not available 2024-04-22T15:04:40.859 INFO:teuthology.orchestra.run.smithi145.stderr:systemctl: disabled 2024-04-22T15:04:40.869 INFO:teuthology.orchestra.run.smithi145.stderr:systemctl: inactive 2024-04-22T15:04:40.869 INFO:teuthology.orchestra.run.smithi145.stderr:firewalld.service is not enabled 2024-04-22T15:04:40.869 INFO:teuthology.orchestra.run.smithi145.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-04-22T15:04:40.869 INFO:teuthology.orchestra.run.smithi145.stderr:Waiting for mgr to start... 2024-04-22T15:04:40.869 INFO:teuthology.orchestra.run.smithi145.stderr:Waiting for mgr... 2024-04-22T15:04:40.994 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:40 smithi145 podman[22116]: 2024-04-22 15:04:40.722052493 +0000 UTC m=+0.313523745 container init 95f50f021d87a9a0c572d17999f705fee933d17dd3d2d43d7e610ea7f4c79dbe (image=quay.io/ceph/ceph:v17.2.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , release=754, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, GIT_BRANCH=HEAD, distribution-scope=public, version=8, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, name=centos-stream, vendor=Red Hat, Inc., io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-22T15:04:40.994 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:40 smithi145 podman[22116]: 2024-04-22 15:04:40.724534981 +0000 UTC m=+0.316006234 container start 95f50f021d87a9a0c572d17999f705fee933d17dd3d2d43d7e610ea7f4c79dbe (image=quay.io/ceph/ceph:v17.2.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y, io.k8s.display-name=CentOS Stream 8, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, io.buildah.version=1.19.8, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, maintainer=Guillaume Abrioux , architecture=x86_64, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, com.redhat.component=centos-stream-container, io.openshift.expose-services=, name=centos-stream, RELEASE=HEAD) 2024-04-22T15:04:40.994 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:40 smithi145 bash[22116]: 95f50f021d87a9a0c572d17999f705fee933d17dd3d2d43d7e610ea7f4c79dbe 2024-04-22T15:04:40.994 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:40 smithi145 systemd[1]: Started Ceph mgr.y for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:04:40.994 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:40.874+0000 7ff5a9294000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T15:04:41.295 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:41.011+0000 7ff5a9294000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T15:04:41.295 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:41.110+0000 7ff5a9294000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T15:04:41.412 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: 2024-04-22T15:04:41.412 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: { 2024-04-22T15:04:41.412 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "fsid": "869afcfa-00b9-11ef-bc93-c7b262605968", 2024-04-22T15:04:41.413 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "health": { 2024-04-22T15:04:41.413 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-22T15:04:41.413 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "checks": {}, 2024-04-22T15:04:41.413 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "mutes": [] 2024-04-22T15:04:41.413 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:41.413 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-22T15:04:41.413 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "quorum": [ 2024-04-22T15:04:41.413 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: 0 2024-04-22T15:04:41.413 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: ], 2024-04-22T15:04:41.413 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-22T15:04:41.413 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "a" 2024-04-22T15:04:41.413 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: ], 2024-04-22T15:04:41.413 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-04-22T15:04:41.414 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "monmap": { 2024-04-22T15:04:41.414 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T15:04:41.414 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-22T15:04:41.414 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-22T15:04:41.414 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:41.414 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "osdmap": { 2024-04-22T15:04:41.414 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T15:04:41.414 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-22T15:04:41.414 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-22T15:04:41.414 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-22T15:04:41.414 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-22T15:04:41.414 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-22T15:04:41.414 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-22T15:04:41.414 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:41.414 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "pgmap": { 2024-04-22T15:04:41.415 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-22T15:04:41.415 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-22T15:04:41.415 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-22T15:04:41.415 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-22T15:04:41.415 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-22T15:04:41.415 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-22T15:04:41.415 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-22T15:04:41.415 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-22T15:04:41.415 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:41.415 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "fsmap": { 2024-04-22T15:04:41.415 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T15:04:41.415 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-22T15:04:41.415 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-22T15:04:41.415 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:41.415 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-22T15:04:41.416 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "available": false, 2024-04-22T15:04:41.416 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-22T15:04:41.416 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "modules": [ 2024-04-22T15:04:41.416 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "iostat", 2024-04-22T15:04:41.416 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "nfs", 2024-04-22T15:04:41.416 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "restful" 2024-04-22T15:04:41.416 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: ], 2024-04-22T15:04:41.416 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "services": {} 2024-04-22T15:04:41.416 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:41.416 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "servicemap": { 2024-04-22T15:04:41.416 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T15:04:41.416 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "modified": "2024-04-22T15:04:33.942626+0000", 2024-04-22T15:04:41.416 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "services": {} 2024-04-22T15:04:41.416 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:41.417 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-22T15:04:41.417 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: } 2024-04-22T15:04:41.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:41 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2290732333' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-22T15:04:41.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:41.434+0000 7ff5a9294000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T15:04:41.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:41.517+0000 7ff5a9294000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T15:04:42.176 INFO:teuthology.orchestra.run.smithi145.stderr:mgr not available, waiting (1/15)... 2024-04-22T15:04:43.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:42 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:42.656+0000 7ff5a9294000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T15:04:43.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:42 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:42.816+0000 7ff5a9294000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T15:04:43.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:42 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:42.897+0000 7ff5a9294000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T15:04:43.392 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:43.208+0000 7ff5a9294000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T15:04:43.392 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:43.297+0000 7ff5a9294000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T15:04:43.392 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:43.388+0000 7ff5a9294000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T15:04:43.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:43.627+0000 7ff5a9294000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T15:04:44.316 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:44 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:44.059+0000 7ff5a9294000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T15:04:44.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:44 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:44.553+0000 7ff5a9294000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T15:04:44.841 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: 2024-04-22T15:04:44.841 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: { 2024-04-22T15:04:44.841 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "fsid": "869afcfa-00b9-11ef-bc93-c7b262605968", 2024-04-22T15:04:44.841 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "health": { 2024-04-22T15:04:44.841 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-22T15:04:44.842 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "checks": {}, 2024-04-22T15:04:44.842 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "mutes": [] 2024-04-22T15:04:44.842 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:44.842 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-22T15:04:44.842 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "quorum": [ 2024-04-22T15:04:44.842 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: 0 2024-04-22T15:04:44.842 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: ], 2024-04-22T15:04:44.842 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-22T15:04:44.842 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "a" 2024-04-22T15:04:44.842 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: ], 2024-04-22T15:04:44.842 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "quorum_age": 5, 2024-04-22T15:04:44.842 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "monmap": { 2024-04-22T15:04:44.842 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T15:04:44.842 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-22T15:04:44.843 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-22T15:04:44.843 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:44.843 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "osdmap": { 2024-04-22T15:04:44.843 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T15:04:44.843 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-22T15:04:44.843 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-22T15:04:44.843 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-22T15:04:44.843 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-22T15:04:44.843 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-22T15:04:44.843 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-22T15:04:44.843 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:44.843 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "pgmap": { 2024-04-22T15:04:44.843 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-22T15:04:44.843 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-22T15:04:44.843 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-22T15:04:44.844 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-22T15:04:44.846 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-22T15:04:44.846 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-22T15:04:44.846 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-22T15:04:44.846 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-22T15:04:44.846 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:44.846 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "fsmap": { 2024-04-22T15:04:44.847 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T15:04:44.847 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-22T15:04:44.847 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-22T15:04:44.847 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:44.847 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-22T15:04:44.847 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "available": false, 2024-04-22T15:04:44.847 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-22T15:04:44.847 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "modules": [ 2024-04-22T15:04:44.847 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "iostat", 2024-04-22T15:04:44.847 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "nfs", 2024-04-22T15:04:44.847 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "restful" 2024-04-22T15:04:44.847 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: ], 2024-04-22T15:04:44.847 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "services": {} 2024-04-22T15:04:44.847 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:44.847 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "servicemap": { 2024-04-22T15:04:44.847 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T15:04:44.848 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "modified": "2024-04-22T15:04:33.942626+0000", 2024-04-22T15:04:44.848 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "services": {} 2024-04-22T15:04:44.848 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:44.848 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-22T15:04:44.848 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: } 2024-04-22T15:04:44.950 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:44 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3425636982' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-22T15:04:44.951 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:44 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:44.646+0000 7ff5a9294000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T15:04:44.951 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:44 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:44.947+0000 7ff5a9294000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T15:04:45.319 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:45.035+0000 7ff5a9294000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T15:04:45.466 INFO:teuthology.orchestra.run.smithi145.stderr:mgr not available, waiting (2/15)... 2024-04-22T15:04:45.576 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:45.441+0000 7ff5a9294000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T15:04:45.576 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:45.573+0000 7ff5a9294000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T15:04:45.866 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:45.664+0000 7ff5a9294000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T15:04:45.866 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:45.776+0000 7ff5a9294000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T15:04:45.866 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:45.862+0000 7ff5a9294000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T15:04:46.758 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:46.398+0000 7ff5a9294000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T15:04:47.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:46.755+0000 7ff5a9294000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T15:04:47.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:46 smithi145 ceph-mon[21899]: Activating manager daemon y 2024-04-22T15:04:47.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:46 smithi145 ceph-mon[21899]: mgrmap e2: y(active, starting, since 0.00285119s) 2024-04-22T15:04:47.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:46 smithi145 ceph-mon[21899]: from='mgr.14100 172.21.15.145:0/1934174902' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:04:47.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:46 smithi145 ceph-mon[21899]: from='mgr.14100 172.21.15.145:0/1934174902' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:04:47.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:46 smithi145 ceph-mon[21899]: from='mgr.14100 172.21.15.145:0/1934174902' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:04:47.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:46 smithi145 ceph-mon[21899]: from='mgr.14100 172.21.15.145:0/1934174902' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:04:47.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:46 smithi145 ceph-mon[21899]: from='mgr.14100 172.21.15.145:0/1934174902' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T15:04:47.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:46 smithi145 ceph-mon[21899]: Manager daemon y is now available 2024-04-22T15:04:47.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:46 smithi145 ceph-mon[21899]: from='mgr.14100 172.21.15.145:0/1934174902' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:04:47.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:46 smithi145 ceph-mon[21899]: from='mgr.14100 172.21.15.145:0/1934174902' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:04:47.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:46 smithi145 ceph-mon[21899]: from='mgr.14100 172.21.15.145:0/1934174902' entity='mgr.y' 2024-04-22T15:04:47.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:46 smithi145 ceph-mon[21899]: from='mgr.14100 172.21.15.145:0/1934174902' entity='mgr.y' 2024-04-22T15:04:47.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:46 smithi145 ceph-mon[21899]: from='mgr.14100 172.21.15.145:0/1934174902' entity='mgr.y' 2024-04-22T15:04:48.143 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: 2024-04-22T15:04:48.143 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: { 2024-04-22T15:04:48.143 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "fsid": "869afcfa-00b9-11ef-bc93-c7b262605968", 2024-04-22T15:04:48.143 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "health": { 2024-04-22T15:04:48.143 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-22T15:04:48.143 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "checks": {}, 2024-04-22T15:04:48.143 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "mutes": [] 2024-04-22T15:04:48.143 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:48.143 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-22T15:04:48.143 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "quorum": [ 2024-04-22T15:04:48.143 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: 0 2024-04-22T15:04:48.144 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: ], 2024-04-22T15:04:48.144 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-22T15:04:48.144 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "a" 2024-04-22T15:04:48.144 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: ], 2024-04-22T15:04:48.144 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "quorum_age": 9, 2024-04-22T15:04:48.144 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "monmap": { 2024-04-22T15:04:48.144 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T15:04:48.144 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-22T15:04:48.144 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-22T15:04:48.144 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:48.144 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "osdmap": { 2024-04-22T15:04:48.144 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T15:04:48.144 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-22T15:04:48.144 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-22T15:04:48.144 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-22T15:04:48.145 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-22T15:04:48.153 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-22T15:04:48.153 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-22T15:04:48.153 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:48.153 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "pgmap": { 2024-04-22T15:04:48.153 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-22T15:04:48.154 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-22T15:04:48.154 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-22T15:04:48.154 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-22T15:04:48.154 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-22T15:04:48.154 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-22T15:04:48.154 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-22T15:04:48.154 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-22T15:04:48.154 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:48.154 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "fsmap": { 2024-04-22T15:04:48.154 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T15:04:48.154 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-22T15:04:48.154 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-22T15:04:48.155 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:48.155 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-22T15:04:48.155 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "available": true, 2024-04-22T15:04:48.155 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-22T15:04:48.155 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "modules": [ 2024-04-22T15:04:48.155 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "iostat", 2024-04-22T15:04:48.155 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "nfs", 2024-04-22T15:04:48.155 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "restful" 2024-04-22T15:04:48.156 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: ], 2024-04-22T15:04:48.156 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "services": {} 2024-04-22T15:04:48.156 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:48.156 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "servicemap": { 2024-04-22T15:04:48.156 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T15:04:48.156 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "modified": "2024-04-22T15:04:33.942626+0000", 2024-04-22T15:04:48.156 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "services": {} 2024-04-22T15:04:48.156 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: }, 2024-04-22T15:04:48.156 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-22T15:04:48.156 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: } 2024-04-22T15:04:48.681 INFO:teuthology.orchestra.run.smithi145.stderr:mgr is available 2024-04-22T15:04:48.804 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:48 smithi145 ceph-mon[21899]: mgrmap e3: y(active, since 1.00611s) 2024-04-22T15:04:48.804 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:48 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/1907013403' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-22T15:04:49.288 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: 2024-04-22T15:04:49.288 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: [global] 2024-04-22T15:04:49.288 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: fsid = 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:04:49.288 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-22T15:04:49.288 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-22T15:04:49.288 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-22T15:04:49.290 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-22T15:04:49.291 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: 2024-04-22T15:04:49.291 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: [mgr] 2024-04-22T15:04:49.291 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-22T15:04:49.291 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: 2024-04-22T15:04:49.291 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: [osd] 2024-04-22T15:04:49.291 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-22T15:04:49.292 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-22T15:04:49.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:49 smithi145 ceph-mon[21899]: mgrmap e4: y(active, since 2s) 2024-04-22T15:04:49.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:49 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3016376729' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-22T15:04:49.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:49 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3016376729' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-22T15:04:49.879 INFO:teuthology.orchestra.run.smithi145.stderr:Enabling cephadm module... 2024-04-22T15:04:51.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:50 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3681218592' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-22T15:04:51.061 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:50 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ignoring --setuser ceph since I am not root 2024-04-22T15:04:51.061 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:50 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ignoring --setgroup ceph since I am not root 2024-04-22T15:04:51.061 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:50 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:50.928+0000 7f82c10d9000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T15:04:51.061 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:51 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:51.056+0000 7f82c10d9000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T15:04:51.317 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:51 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:51.155+0000 7f82c10d9000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T15:04:51.599 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:51 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:51.465+0000 7f82c10d9000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T15:04:51.599 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:51 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:51.548+0000 7f82c10d9000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T15:04:51.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:51 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3681218592' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-22T15:04:51.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:51 smithi145 ceph-mon[21899]: mgrmap e5: y(active, since 4s) 2024-04-22T15:04:52.070 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: { 2024-04-22T15:04:52.070 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "epoch": 5, 2024-04-22T15:04:52.070 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "available": true, 2024-04-22T15:04:52.070 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "active_name": "y", 2024-04-22T15:04:52.070 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-22T15:04:52.070 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: } 2024-04-22T15:04:52.651 INFO:teuthology.orchestra.run.smithi145.stderr:Waiting for the mgr to restart... 2024-04-22T15:04:52.651 INFO:teuthology.orchestra.run.smithi145.stderr:Waiting for mgr epoch 5... 2024-04-22T15:04:52.942 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:52 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/1643849192' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-22T15:04:52.942 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:52 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:52.688+0000 7f82c10d9000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T15:04:52.942 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:52 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:52.853+0000 7f82c10d9000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T15:04:52.942 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:52 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:52.938+0000 7f82c10d9000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T15:04:53.460 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:53.276+0000 7f82c10d9000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T15:04:53.461 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:53.365+0000 7f82c10d9000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T15:04:53.461 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:53.456+0000 7f82c10d9000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T15:04:53.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:53.697+0000 7f82c10d9000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T15:04:54.621 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:54 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:54.128+0000 7f82c10d9000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T15:04:54.622 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:54 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:54.612+0000 7f82c10d9000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T15:04:54.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:54 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:54.705+0000 7f82c10d9000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T15:04:55.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:54 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:54.995+0000 7f82c10d9000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T15:04:55.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:55.084+0000 7f82c10d9000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T15:04:55.826 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:55.485+0000 7f82c10d9000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T15:04:55.826 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:55.616+0000 7f82c10d9000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T15:04:55.826 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:55.707+0000 7f82c10d9000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T15:04:55.826 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:55.821+0000 7f82c10d9000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T15:04:56.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:55.910+0000 7f82c10d9000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T15:04:56.815 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:56 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:56.448+0000 7f82c10d9000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T15:04:57.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:56 smithi145 ceph-mon[21899]: Active manager daemon y restarted 2024-04-22T15:04:57.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:56 smithi145 ceph-mon[21899]: Activating manager daemon y 2024-04-22T15:04:57.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:56 smithi145 ceph-mon[21899]: osdmap e2: 0 total, 0 up, 0 in 2024-04-22T15:04:57.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:56 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:04:56.811+0000 7f82c10d9000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T15:04:57.883 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: { 2024-04-22T15:04:57.883 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-04-22T15:04:57.883 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "initialized": true 2024-04-22T15:04:57.883 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: } 2024-04-22T15:04:58.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:57 smithi145 ceph-mon[21899]: mgrmap e6: y(active, starting, since 0.0524228s) 2024-04-22T15:04:58.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:57 smithi145 ceph-mon[21899]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:04:58.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:57 smithi145 ceph-mon[21899]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T15:04:58.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:57 smithi145 ceph-mon[21899]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:04:58.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:57 smithi145 ceph-mon[21899]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:04:58.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:57 smithi145 ceph-mon[21899]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:04:58.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:57 smithi145 ceph-mon[21899]: Manager daemon y is now available 2024-04-22T15:04:58.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:57 smithi145 ceph-mon[21899]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' 2024-04-22T15:04:58.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:57 smithi145 ceph-mon[21899]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' 2024-04-22T15:04:58.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:57 smithi145 ceph-mon[21899]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:04:58.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:57 smithi145 ceph-mon[21899]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:04:58.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:57 smithi145 ceph-mon[21899]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:04:58.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:57 smithi145 ceph-mon[21899]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:04:58.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:57 smithi145 ceph-mon[21899]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:04:58.466 INFO:teuthology.orchestra.run.smithi145.stderr:mgr epoch 5 is available 2024-04-22T15:04:58.466 INFO:teuthology.orchestra.run.smithi145.stderr:Setting orchestrator backend to cephadm... 2024-04-22T15:04:59.105 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:58 smithi145 ceph-mon[21899]: mgrmap e7: y(active, since 1.05987s) 2024-04-22T15:04:59.106 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:58 smithi145 ceph-mon[21899]: [22/Apr/2024:15:04:58] ENGINE Bus STARTING 2024-04-22T15:04:59.106 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:58 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: [22/Apr/2024:15:04:58] ENGINE Bus STARTING 2024-04-22T15:04:59.106 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:58 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: [22/Apr/2024:15:04:58] ENGINE Serving on https://172.21.15.145:7150 2024-04-22T15:04:59.106 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:04:58 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: [22/Apr/2024:15:04:58] ENGINE Bus STARTED 2024-04-22T15:04:59.964 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:59 smithi145 ceph-mon[21899]: mgrmap e8: y(active, since 2s) 2024-04-22T15:04:59.964 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:59 smithi145 ceph-mon[21899]: [22/Apr/2024:15:04:58] ENGINE Serving on https://172.21.15.145:7150 2024-04-22T15:04:59.964 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:59 smithi145 ceph-mon[21899]: [22/Apr/2024:15:04:58] ENGINE Bus STARTED 2024-04-22T15:04:59.964 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:59 smithi145 ceph-mon[21899]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' 2024-04-22T15:04:59.964 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:59 smithi145 ceph-mon[21899]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:04:59.964 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:59 smithi145 ceph-mon[21899]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:04:59.964 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:59 smithi145 ceph-mon[21899]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' 2024-04-22T15:04:59.965 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:04:59 smithi145 ceph-mon[21899]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:05:00.239 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: value unchanged 2024-04-22T15:05:00.777 INFO:teuthology.orchestra.run.smithi145.stderr:Generating ssh key... 2024-04-22T15:05:01.036 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:00 smithi145 ceph-mon[21899]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:05:01.831 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: Generating public/private rsa key pair. 2024-04-22T15:05:01.831 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: Your identification has been saved in /tmp/tmpndqln2ce/key. 2024-04-22T15:05:01.832 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: Your public key has been saved in /tmp/tmpndqln2ce/key.pub. 2024-04-22T15:05:01.832 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: The key fingerprint is: 2024-04-22T15:05:01.832 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: SHA256:zFivlHtqQDQ/XuvtBm+H5rwVyLv1G2SoW1DNts47jZ0 ceph-869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:05:01.832 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: The key's randomart image is: 2024-04-22T15:05:01.832 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: +---[RSA 3072]----+ 2024-04-22T15:05:01.832 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: | | 2024-04-22T15:05:01.832 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: | o o | 2024-04-22T15:05:01.832 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: | . o. . + | 2024-04-22T15:05:01.832 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: | .=oo....o .| 2024-04-22T15:05:01.832 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: | ...So.oo..+ | 2024-04-22T15:05:01.832 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: | ...oo o.=. | 2024-04-22T15:05:01.832 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: | .o..=.oo*o| 2024-04-22T15:05:01.832 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: | .o..X+oE=| 2024-04-22T15:05:01.832 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: | .. B*o o+| 2024-04-22T15:05:01.832 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: +----[SHA256]-----+ 2024-04-22T15:05:02.661 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:02 smithi145 ceph-mon[21899]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:05:02.662 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:02 smithi145 ceph-mon[21899]: Generating ssh key... 2024-04-22T15:05:02.662 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:02 smithi145 ceph-mon[21899]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' 2024-04-22T15:05:02.662 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:02 smithi145 ceph-mon[21899]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' 2024-04-22T15:05:02.662 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDT+Vy8WEUL+Kz+WRDHgpdDNYFcOv6o0m+CqSl4m/Vqjabql+IIvEgnA1ZMgWFAlUZag4BfmooEiMRucw56Xp+G1RRjbkv+jfQ5YsXQJTC6knb3xHEvqfOcWyk3wrP6kQrcCjovKaA2L628wHUGe3RPGwKQnS6mQ9PQTUSTLozf0YyUi6Ke7AVJvzhaiYdEBOsVC/HbCT6SomlLb/XEmWeXzcVyYdaXFLcnRCBOtiAxAej+ABPw9abiGHagMCElDAmgQkD00Ig3Wk/yYxnpdp2H1RmlWU4UtWp9Yk+b383cyFKr57lHNzrFp00WIoe5GsPDQSMwq/d4Eli582cDxZlQ6/kjoN5qeqfHDqyBqrLSN2k4tArjqUyA/RUIByEzR+vZFND1pvuoxMfIV4WFcoAoqLU3NPGBZdN2ZCfsyPYcSIF1qneaL/3HQxf3fdylebecMDutXF+yLd1d/my6W5q8w3Gls0v/4JTEBD7B0O23r7TKfQMQeBWROfz1l0c5N+0= ceph-869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:05:03.232 INFO:teuthology.orchestra.run.smithi145.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-22T15:05:03.232 INFO:teuthology.orchestra.run.smithi145.stderr:Adding key to root@localhost authorized_keys... 2024-04-22T15:05:03.232 INFO:teuthology.orchestra.run.smithi145.stderr:Adding host smithi145... 2024-04-22T15:05:03.846 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:03 smithi145 ceph-mon[21899]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:05:04.651 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: Added host 'smithi145' with addr '172.21.15.145' 2024-04-22T15:05:05.231 INFO:teuthology.orchestra.run.smithi145.stderr:Deploying unmanaged mon service... 2024-04-22T15:05:05.860 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:05 smithi145 ceph-mon[21899]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi145", "addr": "172.21.15.145", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:05:05.860 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:05 smithi145 ceph-mon[21899]: Deploying cephadm binary to smithi145 2024-04-22T15:05:05.860 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:05 smithi145 ceph-mon[21899]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' 2024-04-22T15:05:05.860 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:05 smithi145 ceph-mon[21899]: Added host smithi145 2024-04-22T15:05:05.860 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:05 smithi145 ceph-mon[21899]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:05:05.860 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: Scheduled mon update... 2024-04-22T15:05:06.572 INFO:teuthology.orchestra.run.smithi145.stderr:Deploying unmanaged mgr service... 2024-04-22T15:05:06.968 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:06 smithi145 ceph-mon[21899]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:05:06.968 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:06 smithi145 ceph-mon[21899]: Saving service mon spec with placement count:5 2024-04-22T15:05:06.968 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:06 smithi145 ceph-mon[21899]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' 2024-04-22T15:05:07.364 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-04-22T15:05:08.615 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:08 smithi145 ceph-mon[21899]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:05:08.616 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:08 smithi145 ceph-mon[21899]: Saving service mgr spec with placement count:2 2024-04-22T15:05:08.616 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:08 smithi145 ceph-mon[21899]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' 2024-04-22T15:05:09.596 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:09 smithi145 ceph-mon[21899]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' 2024-04-22T15:05:09.596 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:09 smithi145 ceph-mon[21899]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' 2024-04-22T15:05:10.854 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:10 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3065499207' entity='client.admin' 2024-04-22T15:05:12.073 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:11 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/1891450196' entity='client.admin' 2024-04-22T15:05:12.073 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:11 smithi145 ceph-mon[21899]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' 2024-04-22T15:05:12.075 INFO:teuthology.orchestra.run.smithi145.stderr:Enabling the dashboard module... 2024-04-22T15:05:13.281 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:13 smithi145 ceph-mon[21899]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' 2024-04-22T15:05:13.282 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:13 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3748984348' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-22T15:05:13.282 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ignoring --setuser ceph since I am not root 2024-04-22T15:05:13.282 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ignoring --setgroup ceph since I am not root 2024-04-22T15:05:13.282 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:05:13.277+0000 7f92a02a2000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T15:05:13.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:05:13.407+0000 7f92a02a2000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T15:05:13.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:05:13.506+0000 7f92a02a2000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T15:05:14.038 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:05:13.816+0000 7f92a02a2000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T15:05:14.039 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:05:13.902+0000 7f92a02a2000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T15:05:14.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:14 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3748984348' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-22T15:05:14.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:14 smithi145 ceph-mon[21899]: mgrmap e9: y(active, since 16s) 2024-04-22T15:05:14.505 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: { 2024-04-22T15:05:14.505 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "epoch": 9, 2024-04-22T15:05:14.505 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "available": true, 2024-04-22T15:05:14.505 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "active_name": "y", 2024-04-22T15:05:14.505 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-22T15:05:14.505 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: } 2024-04-22T15:05:15.300 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:15 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/531773053' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-22T15:05:15.301 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:05:15.048+0000 7f92a02a2000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T15:05:15.301 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:05:15.213+0000 7f92a02a2000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T15:05:15.301 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:05:15.296+0000 7f92a02a2000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T15:05:15.311 INFO:teuthology.orchestra.run.smithi145.stderr:Waiting for the mgr to restart... 2024-04-22T15:05:15.312 INFO:teuthology.orchestra.run.smithi145.stderr:Waiting for mgr epoch 9... 2024-04-22T15:05:15.824 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:05:15.627+0000 7f92a02a2000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T15:05:15.824 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:05:15.720+0000 7f92a02a2000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T15:05:15.824 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:05:15.820+0000 7f92a02a2000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T15:05:16.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:16 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:05:16.072+0000 7f92a02a2000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T15:05:16.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:16 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:05:16.523+0000 7f92a02a2000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T15:05:17.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:05:17.015+0000 7f92a02a2000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T15:05:17.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:05:17.108+0000 7f92a02a2000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T15:05:17.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:05:17.398+0000 7f92a02a2000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T15:05:17.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:05:17.488+0000 7f92a02a2000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T15:05:18.231 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:05:17.892+0000 7f92a02a2000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T15:05:18.232 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:18 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:05:18.022+0000 7f92a02a2000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T15:05:18.232 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:18 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:05:18.113+0000 7f92a02a2000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T15:05:18.232 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:18 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:05:18.227+0000 7f92a02a2000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T15:05:18.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:18 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:05:18.316+0000 7f92a02a2000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T15:05:19.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:18 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:05:18.857+0000 7f92a02a2000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T15:05:19.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:05:19.218+0000 7f92a02a2000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T15:05:20.549 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:20 smithi145 ceph-mon[21899]: Active manager daemon y restarted 2024-04-22T15:05:20.549 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:20 smithi145 ceph-mon[21899]: Activating manager daemon y 2024-04-22T15:05:20.549 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:20 smithi145 ceph-mon[21899]: osdmap e3: 0 total, 0 up, 0 in 2024-04-22T15:05:20.549 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:20 smithi145 ceph-mon[21899]: mgrmap e10: y(active, starting, since 0.650516s) 2024-04-22T15:05:20.550 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:20 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:05:20.550 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:20 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T15:05:20.550 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:20 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:05:20.550 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:20 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:05:20.550 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:20 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:05:20.550 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:20 smithi145 ceph-mon[21899]: Manager daemon y is now available 2024-04-22T15:05:20.550 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:20 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:05:20.550 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:20 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:05:20.550 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:20 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:05:20.550 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:20 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:05:20.884 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: { 2024-04-22T15:05:20.884 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-04-22T15:05:20.884 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: "initialized": true 2024-04-22T15:05:20.884 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: } 2024-04-22T15:05:21.293 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: [22/Apr/2024:15:05:20] ENGINE Bus STARTING 2024-04-22T15:05:21.293 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: [22/Apr/2024:15:05:21] ENGINE Serving on https://172.21.15.145:7150 2024-04-22T15:05:21.293 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:05:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: [22/Apr/2024:15:05:21] ENGINE Bus STARTED 2024-04-22T15:05:21.537 INFO:teuthology.orchestra.run.smithi145.stderr:mgr epoch 9 is available 2024-04-22T15:05:21.538 INFO:teuthology.orchestra.run.smithi145.stderr:Generating a dashboard self-signed certificate... 2024-04-22T15:05:22.148 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:21 smithi145 ceph-mon[21899]: mgrmap e11: y(active, since 1.65296s) 2024-04-22T15:05:22.148 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:21 smithi145 ceph-mon[21899]: [22/Apr/2024:15:05:20] ENGINE Bus STARTING 2024-04-22T15:05:22.148 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:21 smithi145 ceph-mon[21899]: [22/Apr/2024:15:05:21] ENGINE Serving on https://172.21.15.145:7150 2024-04-22T15:05:22.148 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:21 smithi145 ceph-mon[21899]: [22/Apr/2024:15:05:21] ENGINE Bus STARTED 2024-04-22T15:05:22.149 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:21 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:22.149 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:21 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:22.337 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: Self-signed certificate created 2024-04-22T15:05:23.010 INFO:teuthology.orchestra.run.smithi145.stderr:Creating initial admin user... 2024-04-22T15:05:23.269 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:23 smithi145 ceph-mon[21899]: mgrmap e12: y(active, since 2s) 2024-04-22T15:05:23.269 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:23 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:23.269 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:23 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:23.270 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:23 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:23.939 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$I9qCiE/28y8Q5JCnWmKju.XDxV4YvWE3NsITXu8KpBYmbnEfUylcW", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1713798323, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-22T15:05:24.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:24 smithi145 ceph-mon[21899]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:05:24.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:24 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:25.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:25 smithi145 ceph-mon[21899]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:05:25.383 INFO:teuthology.orchestra.run.smithi145.stderr:Fetching dashboard port number... 2024-04-22T15:05:26.011 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: 8443 2024-04-22T15:05:26.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:26 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/1784484035' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-22T15:05:26.733 INFO:teuthology.orchestra.run.smithi145.stderr:systemctl: disabled 2024-04-22T15:05:26.742 INFO:teuthology.orchestra.run.smithi145.stderr:systemctl: inactive 2024-04-22T15:05:26.743 INFO:teuthology.orchestra.run.smithi145.stderr:firewalld.service is not enabled 2024-04-22T15:05:26.743 INFO:teuthology.orchestra.run.smithi145.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-22T15:05:26.746 INFO:teuthology.orchestra.run.smithi145.stderr:Ceph Dashboard is now available at: 2024-04-22T15:05:26.746 INFO:teuthology.orchestra.run.smithi145.stderr: 2024-04-22T15:05:26.746 INFO:teuthology.orchestra.run.smithi145.stderr: URL: https://smithi145:8443/ 2024-04-22T15:05:26.746 INFO:teuthology.orchestra.run.smithi145.stderr: User: admin 2024-04-22T15:05:26.746 INFO:teuthology.orchestra.run.smithi145.stderr: Password: 8prblin0pq 2024-04-22T15:05:26.746 INFO:teuthology.orchestra.run.smithi145.stderr: 2024-04-22T15:05:26.746 INFO:teuthology.orchestra.run.smithi145.stderr:Enabling autotune for osd_memory_target 2024-04-22T15:05:27.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:27 smithi145 ceph-mon[21899]: mgrmap e13: y(active, since 6s) 2024-04-22T15:05:28.687 INFO:teuthology.orchestra.run.smithi145.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-04-22T15:05:29.373 INFO:teuthology.orchestra.run.smithi145.stderr:You can access the Ceph CLI with: 2024-04-22T15:05:29.373 INFO:teuthology.orchestra.run.smithi145.stderr: 2024-04-22T15:05:29.373 INFO:teuthology.orchestra.run.smithi145.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-22T15:05:29.373 INFO:teuthology.orchestra.run.smithi145.stderr: 2024-04-22T15:05:29.373 INFO:teuthology.orchestra.run.smithi145.stderr:Please consider enabling telemetry to help improve Ceph: 2024-04-22T15:05:29.373 INFO:teuthology.orchestra.run.smithi145.stderr: 2024-04-22T15:05:29.373 INFO:teuthology.orchestra.run.smithi145.stderr: ceph telemetry on 2024-04-22T15:05:29.373 INFO:teuthology.orchestra.run.smithi145.stderr: 2024-04-22T15:05:29.373 INFO:teuthology.orchestra.run.smithi145.stderr:For more information see: 2024-04-22T15:05:29.373 INFO:teuthology.orchestra.run.smithi145.stderr: 2024-04-22T15:05:29.373 INFO:teuthology.orchestra.run.smithi145.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-04-22T15:05:29.373 INFO:teuthology.orchestra.run.smithi145.stderr: 2024-04-22T15:05:29.373 INFO:teuthology.orchestra.run.smithi145.stderr:Bootstrap complete. 2024-04-22T15:05:29.406 INFO:tasks.cephadm:Fetching config... 2024-04-22T15:05:29.406 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-04-22T15:05:29.406 DEBUG:teuthology.orchestra.run.smithi145:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-22T15:05:29.422 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-22T15:05:29.423 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-04-22T15:05:29.423 DEBUG:teuthology.orchestra.run.smithi145:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-22T15:05:29.478 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-22T15:05:29.478 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-04-22T15:05:29.478 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd if=/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/mon.a/keyring of=/dev/stdout 2024-04-22T15:05:29.543 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-22T15:05:29.544 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-04-22T15:05:29.544 DEBUG:teuthology.orchestra.run.smithi145:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-22T15:05:29.599 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-22T15:05:29.599 DEBUG:teuthology.orchestra.run.smithi145:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDT+Vy8WEUL+Kz+WRDHgpdDNYFcOv6o0m+CqSl4m/Vqjabql+IIvEgnA1ZMgWFAlUZag4BfmooEiMRucw56Xp+G1RRjbkv+jfQ5YsXQJTC6knb3xHEvqfOcWyk3wrP6kQrcCjovKaA2L628wHUGe3RPGwKQnS6mQ9PQTUSTLozf0YyUi6Ke7AVJvzhaiYdEBOsVC/HbCT6SomlLb/XEmWeXzcVyYdaXFLcnRCBOtiAxAej+ABPw9abiGHagMCElDAmgQkD00Ig3Wk/yYxnpdp2H1RmlWU4UtWp9Yk+b383cyFKr57lHNzrFp00WIoe5GsPDQSMwq/d4Eli582cDxZlQ6/kjoN5qeqfHDqyBqrLSN2k4tArjqUyA/RUIByEzR+vZFND1pvuoxMfIV4WFcoAoqLU3NPGBZdN2ZCfsyPYcSIF1qneaL/3HQxf3fdylebecMDutXF+yLd1d/my6W5q8w3Gls0v/4JTEBD7B0O23r7TKfQMQeBWROfz1l0c5N+0= ceph-869afcfa-00b9-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-22T15:05:29.670 INFO:teuthology.orchestra.run.smithi145.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDT+Vy8WEUL+Kz+WRDHgpdDNYFcOv6o0m+CqSl4m/Vqjabql+IIvEgnA1ZMgWFAlUZag4BfmooEiMRucw56Xp+G1RRjbkv+jfQ5YsXQJTC6knb3xHEvqfOcWyk3wrP6kQrcCjovKaA2L628wHUGe3RPGwKQnS6mQ9PQTUSTLozf0YyUi6Ke7AVJvzhaiYdEBOsVC/HbCT6SomlLb/XEmWeXzcVyYdaXFLcnRCBOtiAxAej+ABPw9abiGHagMCElDAmgQkD00Ig3Wk/yYxnpdp2H1RmlWU4UtWp9Yk+b383cyFKr57lHNzrFp00WIoe5GsPDQSMwq/d4Eli582cDxZlQ6/kjoN5qeqfHDqyBqrLSN2k4tArjqUyA/RUIByEzR+vZFND1pvuoxMfIV4WFcoAoqLU3NPGBZdN2ZCfsyPYcSIF1qneaL/3HQxf3fdylebecMDutXF+yLd1d/my6W5q8w3Gls0v/4JTEBD7B0O23r7TKfQMQeBWROfz1l0c5N+0= ceph-869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:05:29.681 DEBUG:teuthology.orchestra.run.smithi172:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDT+Vy8WEUL+Kz+WRDHgpdDNYFcOv6o0m+CqSl4m/Vqjabql+IIvEgnA1ZMgWFAlUZag4BfmooEiMRucw56Xp+G1RRjbkv+jfQ5YsXQJTC6knb3xHEvqfOcWyk3wrP6kQrcCjovKaA2L628wHUGe3RPGwKQnS6mQ9PQTUSTLozf0YyUi6Ke7AVJvzhaiYdEBOsVC/HbCT6SomlLb/XEmWeXzcVyYdaXFLcnRCBOtiAxAej+ABPw9abiGHagMCElDAmgQkD00Ig3Wk/yYxnpdp2H1RmlWU4UtWp9Yk+b383cyFKr57lHNzrFp00WIoe5GsPDQSMwq/d4Eli582cDxZlQ6/kjoN5qeqfHDqyBqrLSN2k4tArjqUyA/RUIByEzR+vZFND1pvuoxMfIV4WFcoAoqLU3NPGBZdN2ZCfsyPYcSIF1qneaL/3HQxf3fdylebecMDutXF+yLd1d/my6W5q8w3Gls0v/4JTEBD7B0O23r7TKfQMQeBWROfz1l0c5N+0= ceph-869afcfa-00b9-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-22T15:05:29.720 INFO:teuthology.orchestra.run.smithi172.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDT+Vy8WEUL+Kz+WRDHgpdDNYFcOv6o0m+CqSl4m/Vqjabql+IIvEgnA1ZMgWFAlUZag4BfmooEiMRucw56Xp+G1RRjbkv+jfQ5YsXQJTC6knb3xHEvqfOcWyk3wrP6kQrcCjovKaA2L628wHUGe3RPGwKQnS6mQ9PQTUSTLozf0YyUi6Ke7AVJvzhaiYdEBOsVC/HbCT6SomlLb/XEmWeXzcVyYdaXFLcnRCBOtiAxAej+ABPw9abiGHagMCElDAmgQkD00Ig3Wk/yYxnpdp2H1RmlWU4UtWp9Yk+b383cyFKr57lHNzrFp00WIoe5GsPDQSMwq/d4Eli582cDxZlQ6/kjoN5qeqfHDqyBqrLSN2k4tArjqUyA/RUIByEzR+vZFND1pvuoxMfIV4WFcoAoqLU3NPGBZdN2ZCfsyPYcSIF1qneaL/3HQxf3fdylebecMDutXF+yLd1d/my6W5q8w3Gls0v/4JTEBD7B0O23r7TKfQMQeBWROfz1l0c5N+0= ceph-869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:05:29.731 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-22T15:05:29.910 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:29 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2882001211' entity='client.admin' 2024-04-22T15:05:33.422 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-22T15:05:33.422 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-22T15:05:34.119 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:33 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/537971518' entity='client.admin' 2024-04-22T15:05:35.807 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:35 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:35.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:35 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi145", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:05:35.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:35 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:35.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:35 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:05:35.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:35 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:36.358 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi172 2024-04-22T15:05:36.358 DEBUG:teuthology.orchestra.run.smithi172:> set -ex 2024-04-22T15:05:36.358 DEBUG:teuthology.orchestra.run.smithi172:> dd of=/etc/ceph/ceph.conf 2024-04-22T15:05:36.377 DEBUG:teuthology.orchestra.run.smithi172:> set -ex 2024-04-22T15:05:36.377 DEBUG:teuthology.orchestra.run.smithi172:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:05:36.435 INFO:tasks.cephadm:Adding host smithi172 to orchestrator... 2024-04-22T15:05:36.435 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph orch host add smithi172 2024-04-22T15:05:37.057 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:36 smithi145 ceph-mon[21899]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:05:37.057 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:36 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:37.057 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:36 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:05:37.057 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:36 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:05:37.057 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:36 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:05:37.057 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:36 smithi145 ceph-mon[21899]: Updating smithi145:/etc/ceph/ceph.conf 2024-04-22T15:05:37.057 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:36 smithi145 ceph-mon[21899]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:05:37.057 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:36 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:37.057 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:36 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:39.119 INFO:teuthology.orchestra.run.smithi145.stdout:Added host 'smithi172' with addr '172.21.15.172' 2024-04-22T15:05:39.706 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph orch host ls --format=json 2024-04-22T15:05:40.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:40 smithi145 ceph-mon[21899]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi172", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:05:40.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:40 smithi145 ceph-mon[21899]: Deploying cephadm binary to smithi172 2024-04-22T15:05:40.324 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:40 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:40.324 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:40 smithi145 ceph-mon[21899]: Added host smithi172 2024-04-22T15:05:40.324 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:40 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:05:40.324 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:40 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:05:40.324 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:40 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:05:40.324 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:40 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:40.325 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:40 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:41.246 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:41 smithi145 ceph-mon[21899]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:41.247 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:41 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:41.641 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-04-22T15:05:41.641 INFO:teuthology.orchestra.run.smithi145.stdout:[{"addr": "172.21.15.145", "hostname": "smithi145", "labels": [], "status": ""}, {"addr": "172.21.15.172", "hostname": "smithi172", "labels": [], "status": ""}] 2024-04-22T15:05:42.304 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-22T15:05:42.305 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph osd crush tunables default 2024-04-22T15:05:43.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:43 smithi145 ceph-mon[21899]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T15:05:43.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:43 smithi145 ceph-mon[21899]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:44.886 INFO:teuthology.orchestra.run.smithi145.stderr:adjusted tunables profile to default 2024-04-22T15:05:44.899 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:44 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:44.899 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:44 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:44.899 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:44 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi172", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:05:44.899 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:44 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:44.900 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:44 smithi145 ceph-mon[21899]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:44.900 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:44 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3407015052' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-22T15:05:45.538 INFO:tasks.cephadm:Adding mon.a on smithi145 2024-04-22T15:05:45.538 INFO:tasks.cephadm:Adding mon.c on smithi145 2024-04-22T15:05:45.538 INFO:tasks.cephadm:Adding mon.b on smithi172 2024-04-22T15:05:45.538 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph orch apply mon '3;smithi145:172.21.15.145=a;smithi145:[v2:172.21.15.145:3301,v1:172.21.15.145:6790]=c;smithi172:172.21.15.172=b' 2024-04-22T15:05:46.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:45 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3407015052' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-22T15:05:46.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:45 smithi145 ceph-mon[21899]: osdmap e4: 0 total, 0 up, 0 in 2024-04-22T15:05:47.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:46 smithi145 ceph-mon[21899]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:47.386 INFO:teuthology.orchestra.run.smithi172.stdout:Scheduled mon update... 2024-04-22T15:05:48.081 DEBUG:teuthology.orchestra.run.smithi145:mon.c> sudo journalctl -f -n 0 -u ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mon.c.service 2024-04-22T15:05:48.084 DEBUG:teuthology.orchestra.run.smithi172:mon.b> sudo journalctl -f -n 0 -u ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mon.b.service 2024-04-22T15:05:48.086 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-22T15:05:48.086 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T15:05:48.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:48 smithi145 ceph-mon[21899]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi145:172.21.15.145=a;smithi145:[v2:172.21.15.145:3301,v1:172.21.15.145:6790]=c;smithi172:172.21.15.172=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:05:48.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:48 smithi145 ceph-mon[21899]: Saving service mon spec with placement smithi145:172.21.15.145=a;smithi145:[v2:172.21.15.145:3301,v1:172.21.15.145:6790]=c;smithi172:172.21.15.172=b;count:3 2024-04-22T15:05:48.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:48 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:48.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:48 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:05:48.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:48 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:05:48.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:48 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:05:48.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:48 smithi145 ceph-mon[21899]: Updating smithi172:/etc/ceph/ceph.conf 2024-04-22T15:05:48.875 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:48 smithi145 ceph-mon[21899]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:05:48.875 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:48 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:48.875 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:48 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:48.875 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:48 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:05:48.875 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:48 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:05:48.875 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:48 smithi145 ceph-mon[21899]: Deploying daemon mon.c on smithi145 2024-04-22T15:05:48.875 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:48 smithi145 ceph-mon[21899]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:49.963 INFO:teuthology.orchestra.run.smithi172.stdout: 2024-04-22T15:05:49.963 INFO:teuthology.orchestra.run.smithi172.stdout:{"epoch":1,"fsid":"869afcfa-00b9-11ef-bc93-c7b262605968","modified":"2024-04-22T15:04:30.924274Z","created":"2024-04-22T15:04:30.924274Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:3300","nonce":0},{"type":"v1","addr":"172.21.15.145:6789","nonce":0}]},"addr":"172.21.15.145:6789/0","public_addr":"172.21.15.145:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-22T15:05:49.965 INFO:teuthology.orchestra.run.smithi172.stderr:dumped monmap epoch 1 2024-04-22T15:05:50.932 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:50 smithi145 podman[26588]: 2024-04-22 15:05:50.853247017 +0000 UTC m=+0.119708249 container create b71d9b9f850909b4f0dbe9e34be195dea8120952106575a706c786235bbb4642 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-c, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, version=8, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, release=754, distribution-scope=public, architecture=x86_64, ceph=True, vcs-type=git, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream) 2024-04-22T15:05:51.189 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:50 smithi145 ceph-mon[21899]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:51.190 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:50 smithi145 ceph-mon[21899]: from='client.? 172.21.15.172:0/3177305875' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T15:05:51.191 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 podman[26588]: 2024-04-22 15:05:51.055555136 +0000 UTC m=+0.322016368 container init b71d9b9f850909b4f0dbe9e34be195dea8120952106575a706c786235bbb4642 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-c, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, version=8, architecture=x86_64, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, vendor=Red Hat, Inc., io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container) 2024-04-22T15:05:51.191 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 podman[26588]: 2024-04-22 15:05:51.058925205 +0000 UTC m=+0.325386437 container start b71d9b9f850909b4f0dbe9e34be195dea8120952106575a706c786235bbb4642 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-c, ceph=True, io.buildah.version=1.19.8, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, name=centos-stream, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, vcs-type=git, distribution-scope=public, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, version=8, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-22T15:05:51.191 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: set uid:gid to 167:167 (ceph:ceph) 2024-04-22T15:05:51.192 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-22T15:05:51.192 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: pidfile_write: ignore empty --pid-file 2024-04-22T15:05:51.192 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: load: jerasure load: lrc 2024-04-22T15:05:51.192 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: RocksDB version: 6.15.5 2024-04-22T15:05:51.192 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-22T15:05:51.192 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Compile date Apr 18 2022 2024-04-22T15:05:51.192 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: DB SUMMARY 2024-04-22T15:05:51.192 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: DB Session ID: EGI8X7I2VLGSEZDJOWXA 2024-04-22T15:05:51.192 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: CURRENT file: CURRENT 2024-04-22T15:05:51.192 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: IDENTITY file: IDENTITY 2024-04-22T15:05:51.192 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-04-22T15:05:51.192 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-04-22T15:05:51.193 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 511 ; 2024-04-22T15:05:51.193 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.error_if_exists: 0 2024-04-22T15:05:51.193 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.create_if_missing: 0 2024-04-22T15:05:51.193 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.paranoid_checks: 1 2024-04-22T15:05:51.193 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-22T15:05:51.193 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.env: 0x5596498c6860 2024-04-22T15:05:51.193 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.fs: Posix File System 2024-04-22T15:05:51.193 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.info_log: 0x55964b74ddc0 2024-04-22T15:05:51.193 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.max_file_opening_threads: 16 2024-04-22T15:05:51.193 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.statistics: (nil) 2024-04-22T15:05:51.193 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.use_fsync: 0 2024-04-22T15:05:51.193 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.max_log_file_size: 0 2024-04-22T15:05:51.194 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-22T15:05:51.194 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-22T15:05:51.194 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.keep_log_file_num: 1000 2024-04-22T15:05:51.194 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.recycle_log_file_num: 0 2024-04-22T15:05:51.194 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.allow_fallocate: 1 2024-04-22T15:05:51.194 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.allow_mmap_reads: 0 2024-04-22T15:05:51.194 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.allow_mmap_writes: 0 2024-04-22T15:05:51.194 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.use_direct_reads: 0 2024-04-22T15:05:51.194 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-22T15:05:51.195 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.create_missing_column_families: 0 2024-04-22T15:05:51.195 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.db_log_dir: 2024-04-22T15:05:51.195 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-04-22T15:05:51.195 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-22T15:05:51.195 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-22T15:05:51.195 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-22T15:05:51.195 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-22T15:05:51.195 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-22T15:05:51.195 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-22T15:05:51.195 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.advise_random_on_open: 1 2024-04-22T15:05:51.195 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.db_write_buffer_size: 0 2024-04-22T15:05:51.195 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.write_buffer_manager: 0x55964b83e270 2024-04-22T15:05:51.196 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-22T15:05:51.196 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-22T15:05:51.196 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-22T15:05:51.196 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-22T15:05:51.196 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.rate_limiter: (nil) 2024-04-22T15:05:51.196 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-22T15:05:51.196 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.wal_recovery_mode: 2 2024-04-22T15:05:51.196 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.enable_thread_tracking: 0 2024-04-22T15:05:51.196 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.enable_pipelined_write: 0 2024-04-22T15:05:51.196 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.unordered_write: 0 2024-04-22T15:05:51.196 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-22T15:05:51.196 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-22T15:05:51.196 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-22T15:05:51.197 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-22T15:05:51.197 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.row_cache: None 2024-04-22T15:05:51.197 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.wal_filter: None 2024-04-22T15:05:51.197 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-22T15:05:51.197 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.allow_ingest_behind: 0 2024-04-22T15:05:51.197 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.preserve_deletes: 0 2024-04-22T15:05:51.197 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.two_write_queues: 0 2024-04-22T15:05:51.197 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.manual_wal_flush: 0 2024-04-22T15:05:51.197 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.atomic_flush: 0 2024-04-22T15:05:51.197 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-22T15:05:51.197 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-22T15:05:51.197 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-22T15:05:51.198 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.log_readahead_size: 0 2024-04-22T15:05:51.198 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-22T15:05:51.198 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.best_efforts_recovery: 0 2024-04-22T15:05:51.198 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-22T15:05:51.198 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-22T15:05:51.198 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.allow_data_in_errors: 0 2024-04-22T15:05:51.198 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.db_host_id: __hostname__ 2024-04-22T15:05:51.198 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.max_background_jobs: 2 2024-04-22T15:05:51.198 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.max_background_compactions: -1 2024-04-22T15:05:51.198 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.max_subcompactions: 1 2024-04-22T15:05:51.198 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-22T15:05:51.198 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-22T15:05:51.198 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-22T15:05:51.199 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.max_total_wal_size: 0 2024-04-22T15:05:51.199 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-22T15:05:51.199 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-22T15:05:51.199 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-22T15:05:51.199 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-22T15:05:51.199 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.max_open_files: -1 2024-04-22T15:05:51.199 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.bytes_per_sync: 0 2024-04-22T15:05:51.199 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-22T15:05:51.199 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-22T15:05:51.199 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.compaction_readahead_size: 0 2024-04-22T15:05:51.199 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.max_background_flushes: -1 2024-04-22T15:05:51.199 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Compression algorithms supported: 2024-04-22T15:05:51.199 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-22T15:05:51.200 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: kZSTD supported: 0 2024-04-22T15:05:51.200 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: kXpressCompression supported: 0 2024-04-22T15:05:51.200 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: kLZ4HCCompression supported: 1 2024-04-22T15:05:51.200 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: kLZ4Compression supported: 1 2024-04-22T15:05:51.200 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: kBZip2Compression supported: 0 2024-04-22T15:05:51.200 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: kZlibCompression supported: 1 2024-04-22T15:05:51.200 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: kSnappyCompression supported: 1 2024-04-22T15:05:51.200 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-22T15:05:51.200 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-04-22T15:05:51.200 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-22T15:05:51.200 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-22T15:05:51.200 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.merge_operator: 2024-04-22T15:05:51.201 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.compaction_filter: None 2024-04-22T15:05:51.201 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.compaction_filter_factory: None 2024-04-22T15:05:51.201 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.sst_partitioner_factory: None 2024-04-22T15:05:51.201 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-22T15:05:51.201 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-22T15:05:51.201 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55964b719d00) 2024-04-22T15:05:51.201 INFO:journalctl@ceph.mon.c.smithi145.stdout: cache_index_and_filter_blocks: 1 2024-04-22T15:05:51.201 INFO:journalctl@ceph.mon.c.smithi145.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-22T15:05:51.201 INFO:journalctl@ceph.mon.c.smithi145.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-22T15:05:51.201 INFO:journalctl@ceph.mon.c.smithi145.stdout: pin_top_level_index_and_filter: 1 2024-04-22T15:05:51.201 INFO:journalctl@ceph.mon.c.smithi145.stdout: index_type: 0 2024-04-22T15:05:51.201 INFO:journalctl@ceph.mon.c.smithi145.stdout: data_block_index_type: 0 2024-04-22T15:05:51.201 INFO:journalctl@ceph.mon.c.smithi145.stdout: index_shortening: 1 2024-04-22T15:05:51.202 INFO:journalctl@ceph.mon.c.smithi145.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-22T15:05:51.202 INFO:journalctl@ceph.mon.c.smithi145.stdout: hash_index_allow_collision: 1 2024-04-22T15:05:51.202 INFO:journalctl@ceph.mon.c.smithi145.stdout: checksum: 1 2024-04-22T15:05:51.202 INFO:journalctl@ceph.mon.c.smithi145.stdout: no_block_cache: 0 2024-04-22T15:05:51.202 INFO:journalctl@ceph.mon.c.smithi145.stdout: block_cache: 0x55964b784170 2024-04-22T15:05:51.202 INFO:journalctl@ceph.mon.c.smithi145.stdout: block_cache_name: BinnedLRUCache 2024-04-22T15:05:51.202 INFO:journalctl@ceph.mon.c.smithi145.stdout: block_cache_options: 2024-04-22T15:05:51.202 INFO:journalctl@ceph.mon.c.smithi145.stdout: capacity : 536870912 2024-04-22T15:05:51.202 INFO:journalctl@ceph.mon.c.smithi145.stdout: num_shard_bits : 4 2024-04-22T15:05:51.202 INFO:journalctl@ceph.mon.c.smithi145.stdout: strict_capacity_limit : 0 2024-04-22T15:05:51.202 INFO:journalctl@ceph.mon.c.smithi145.stdout: high_pri_pool_ratio: 0.000 2024-04-22T15:05:51.202 INFO:journalctl@ceph.mon.c.smithi145.stdout: block_cache_compressed: (nil) 2024-04-22T15:05:51.203 INFO:journalctl@ceph.mon.c.smithi145.stdout: persistent_cache: (nil) 2024-04-22T15:05:51.203 INFO:journalctl@ceph.mon.c.smithi145.stdout: block_size: 4096 2024-04-22T15:05:51.203 INFO:journalctl@ceph.mon.c.smithi145.stdout: block_size_deviation: 10 2024-04-22T15:05:51.203 INFO:journalctl@ceph.mon.c.smithi145.stdout: block_restart_interval: 16 2024-04-22T15:05:51.203 INFO:journalctl@ceph.mon.c.smithi145.stdout: index_block_restart_interval: 1 2024-04-22T15:05:51.203 INFO:journalctl@ceph.mon.c.smithi145.stdout: metadata_block_size: 4096 2024-04-22T15:05:51.203 INFO:journalctl@ceph.mon.c.smithi145.stdout: partition_filters: 0 2024-04-22T15:05:51.203 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_delta_encoding: 1 2024-04-22T15:05:51.203 INFO:journalctl@ceph.mon.c.smithi145.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-22T15:05:51.203 INFO:journalctl@ceph.mon.c.smithi145.stdout: whole_key_filtering: 1 2024-04-22T15:05:51.203 INFO:journalctl@ceph.mon.c.smithi145.stdout: verify_compression: 0 2024-04-22T15:05:51.203 INFO:journalctl@ceph.mon.c.smithi145.stdout: read_amp_bytes_per_bit: 0 2024-04-22T15:05:51.203 INFO:journalctl@ceph.mon.c.smithi145.stdout: format_version: 4 2024-04-22T15:05:51.204 INFO:journalctl@ceph.mon.c.smithi145.stdout: enable_index_compression: 1 2024-04-22T15:05:51.204 INFO:journalctl@ceph.mon.c.smithi145.stdout: block_align: 0 2024-04-22T15:05:51.204 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.write_buffer_size: 33554432 2024-04-22T15:05:51.204 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.max_write_buffer_number: 2 2024-04-22T15:05:51.204 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.compression: NoCompression 2024-04-22T15:05:51.204 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.bottommost_compression: Disabled 2024-04-22T15:05:51.204 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.prefix_extractor: nullptr 2024-04-22T15:05:51.204 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-22T15:05:51.204 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.num_levels: 7 2024-04-22T15:05:51.204 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-22T15:05:51.204 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-22T15:05:51.204 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-22T15:05:51.204 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-22T15:05:51.205 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-22T15:05:51.205 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-22T15:05:51.205 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-22T15:05:51.205 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-22T15:05:51.205 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-22T15:05:51.205 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-22T15:05:51.205 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-22T15:05:51.205 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.compression_opts.level: 32767 2024-04-22T15:05:51.205 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.compression_opts.strategy: 0 2024-04-22T15:05:51.205 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-22T15:05:51.205 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-22T15:05:51.205 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-22T15:05:51.205 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.compression_opts.enabled: false 2024-04-22T15:05:51.206 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-22T15:05:51.206 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-22T15:05:51.206 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-22T15:05:51.206 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.target_file_size_base: 67108864 2024-04-22T15:05:51.206 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-22T15:05:51.206 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-22T15:05:51.206 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-22T15:05:51.206 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-22T15:05:51.206 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-22T15:05:51.206 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-22T15:05:51.206 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-22T15:05:51.206 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-22T15:05:51.207 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-22T15:05:51.207 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-22T15:05:51.207 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-22T15:05:51.207 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-22T15:05:51.207 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-22T15:05:51.207 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.arena_block_size: 4194304 2024-04-22T15:05:51.207 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-22T15:05:51.207 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-22T15:05:51.207 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-22T15:05:51.207 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.disable_auto_compactions: 0 2024-04-22T15:05:51.207 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-22T15:05:51.207 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-22T15:05:51.207 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-22T15:05:51.208 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-22T15:05:51.208 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-22T15:05:51.208 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-22T15:05:51.208 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-22T15:05:51.208 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-22T15:05:51.208 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-22T15:05:51.208 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-22T15:05:51.208 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.table_properties_collectors: 2024-04-22T15:05:51.208 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.inplace_update_support: 0 2024-04-22T15:05:51.208 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-22T15:05:51.208 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-22T15:05:51.208 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-22T15:05:51.208 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-22T15:05:51.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.bloom_locality: 0 2024-04-22T15:05:51.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.max_successive_merges: 0 2024-04-22T15:05:51.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-22T15:05:51.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.paranoid_file_checks: 0 2024-04-22T15:05:51.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.force_consistency_checks: 1 2024-04-22T15:05:51.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.report_bg_io_stats: 0 2024-04-22T15:05:51.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.ttl: 2592000 2024-04-22T15:05:51.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-22T15:05:51.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.enable_blob_files: false 2024-04-22T15:05:51.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.min_blob_size: 0 2024-04-22T15:05:51.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.blob_file_size: 268435456 2024-04-22T15:05:51.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-22T15:05:51.210 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-22T15:05:51.210 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-22T15:05:51.210 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-22T15:05:51.210 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-04-22T15:05:51.210 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-04-22T15:05:51.210 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713798351084587, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-22T15:05:51.210 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-04-22T15:05:51.210 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-22T15:05:51.210 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713798351084880, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1713798351, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "43454046-25dc-419a-94a6-87934881bebd", "db_session_id": "EGI8X7I2VLGSEZDJOWXA"}} 2024-04-22T15:05:51.210 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-04-22T15:05:51.210 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713798351085184, "job": 1, "event": "recovery_finished"} 2024-04-22T15:05:51.210 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-22T15:05:51.211 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55964b76aa80 2024-04-22T15:05:51.211 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: DB pointer 0x55964b77a000 2024-04-22T15:05:51.211 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-22T15:05:51.211 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-22T15:05:51.211 INFO:journalctl@ceph.mon.c.smithi145.stdout: ** DB Stats ** 2024-04-22T15:05:51.211 INFO:journalctl@ceph.mon.c.smithi145.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T15:05:51.211 INFO:journalctl@ceph.mon.c.smithi145.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-22T15:05:51.211 INFO:journalctl@ceph.mon.c.smithi145.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-22T15:05:51.211 INFO:journalctl@ceph.mon.c.smithi145.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T15:05:51.211 INFO:journalctl@ceph.mon.c.smithi145.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-22T15:05:51.211 INFO:journalctl@ceph.mon.c.smithi145.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-22T15:05:51.211 INFO:journalctl@ceph.mon.c.smithi145.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T15:05:51.211 INFO:journalctl@ceph.mon.c.smithi145.stdout: 2024-04-22T15:05:51.212 INFO:journalctl@ceph.mon.c.smithi145.stdout: ** Compaction Stats [default] ** 2024-04-22T15:05:51.212 INFO:journalctl@ceph.mon.c.smithi145.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T15:05:51.212 INFO:journalctl@ceph.mon.c.smithi145.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T15:05:51.212 INFO:journalctl@ceph.mon.c.smithi145.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-04-22T15:05:51.212 INFO:journalctl@ceph.mon.c.smithi145.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-04-22T15:05:51.212 INFO:journalctl@ceph.mon.c.smithi145.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-04-22T15:05:51.212 INFO:journalctl@ceph.mon.c.smithi145.stdout: 2024-04-22T15:05:51.212 INFO:journalctl@ceph.mon.c.smithi145.stdout: ** Compaction Stats [default] ** 2024-04-22T15:05:51.212 INFO:journalctl@ceph.mon.c.smithi145.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T15:05:51.212 INFO:journalctl@ceph.mon.c.smithi145.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T15:05:51.212 INFO:journalctl@ceph.mon.c.smithi145.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-04-22T15:05:51.212 INFO:journalctl@ceph.mon.c.smithi145.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T15:05:51.212 INFO:journalctl@ceph.mon.c.smithi145.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-22T15:05:51.213 INFO:journalctl@ceph.mon.c.smithi145.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-22T15:05:51.213 INFO:journalctl@ceph.mon.c.smithi145.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-22T15:05:51.213 INFO:journalctl@ceph.mon.c.smithi145.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-22T15:05:51.213 INFO:journalctl@ceph.mon.c.smithi145.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-22T15:05:51.213 INFO:journalctl@ceph.mon.c.smithi145.stdout: Cumulative compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T15:05:51.213 INFO:journalctl@ceph.mon.c.smithi145.stdout: Interval compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T15:05:51.213 INFO:journalctl@ceph.mon.c.smithi145.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-22T15:05:51.213 INFO:journalctl@ceph.mon.c.smithi145.stdout: 2024-04-22T15:05:51.213 INFO:journalctl@ceph.mon.c.smithi145.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-22T15:05:51.213 INFO:journalctl@ceph.mon.c.smithi145.stdout: 2024-04-22T15:05:51.213 INFO:journalctl@ceph.mon.c.smithi145.stdout: ** Compaction Stats [default] ** 2024-04-22T15:05:51.213 INFO:journalctl@ceph.mon.c.smithi145.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T15:05:51.213 INFO:journalctl@ceph.mon.c.smithi145.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T15:05:51.214 INFO:journalctl@ceph.mon.c.smithi145.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-04-22T15:05:51.214 INFO:journalctl@ceph.mon.c.smithi145.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-04-22T15:05:51.214 INFO:journalctl@ceph.mon.c.smithi145.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-22T15:05:51.214 INFO:journalctl@ceph.mon.c.smithi145.stdout: 2024-04-22T15:05:51.214 INFO:journalctl@ceph.mon.c.smithi145.stdout: ** Compaction Stats [default] ** 2024-04-22T15:05:51.214 INFO:journalctl@ceph.mon.c.smithi145.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T15:05:51.214 INFO:journalctl@ceph.mon.c.smithi145.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T15:05:51.214 INFO:journalctl@ceph.mon.c.smithi145.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-04-22T15:05:51.214 INFO:journalctl@ceph.mon.c.smithi145.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T15:05:51.214 INFO:journalctl@ceph.mon.c.smithi145.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-22T15:05:51.214 INFO:journalctl@ceph.mon.c.smithi145.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-22T15:05:51.214 INFO:journalctl@ceph.mon.c.smithi145.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-22T15:05:51.214 INFO:journalctl@ceph.mon.c.smithi145.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-22T15:05:51.215 INFO:journalctl@ceph.mon.c.smithi145.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-22T15:05:51.215 INFO:journalctl@ceph.mon.c.smithi145.stdout: Cumulative compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T15:05:51.215 INFO:journalctl@ceph.mon.c.smithi145.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T15:05:51.215 INFO:journalctl@ceph.mon.c.smithi145.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-22T15:05:51.215 INFO:journalctl@ceph.mon.c.smithi145.stdout: 2024-04-22T15:05:51.215 INFO:journalctl@ceph.mon.c.smithi145.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-22T15:05:51.215 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-04-22T15:05:51.215 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: using public_addrv [v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0] 2024-04-22T15:05:51.216 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: starting mon.c rank -1 at public addrs [v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0] at bind addrs [v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:05:51.216 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mon.c@-1(???) e0 preinit fsid 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:05:51.217 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mon.c@-1(synchronizing).mds e1 new map 2024-04-22T15:05:51.217 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mon.c@-1(synchronizing).mds e1 print_map 2024-04-22T15:05:51.217 INFO:journalctl@ceph.mon.c.smithi145.stdout: e1 2024-04-22T15:05:51.217 INFO:journalctl@ceph.mon.c.smithi145.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-22T15:05:51.217 INFO:journalctl@ceph.mon.c.smithi145.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-22T15:05:51.217 INFO:journalctl@ceph.mon.c.smithi145.stdout: legacy client fscid: -1 2024-04-22T15:05:51.217 INFO:journalctl@ceph.mon.c.smithi145.stdout: 2024-04-22T15:05:51.217 INFO:journalctl@ceph.mon.c.smithi145.stdout: No filesystems configured 2024-04-22T15:05:51.217 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-22T15:05:51.217 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-22T15:05:51.217 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-22T15:05:51.217 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-22T15:05:51.218 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-22T15:05:51.218 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-22T15:05:51.218 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-22T15:05:51.218 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T15:05:51.218 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T15:05:51.218 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T15:05:51.218 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mkfs 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:05:51.218 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-22T15:05:51.218 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-22T15:05:51.218 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: monmap e1: 1 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0]} 2024-04-22T15:05:51.218 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: fsmap 2024-04-22T15:05:51.218 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: osdmap e1: 0 total, 0 up, 0 in 2024-04-22T15:05:51.218 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mgrmap e1: no daemons active 2024-04-22T15:05:51.219 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2958196393' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T15:05:51.219 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/1086624769' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-22T15:05:51.219 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/1086624769' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-22T15:05:51.219 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2652657102' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:05:51.219 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-22T15:05:51.219 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: monmap e1: 1 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0]} 2024-04-22T15:05:51.219 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: fsmap 2024-04-22T15:05:51.219 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: osdmap e1: 0 total, 0 up, 0 in 2024-04-22T15:05:51.219 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mgrmap e1: no daemons active 2024-04-22T15:05:51.219 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2045584094' entity='client.admin' 2024-04-22T15:05:51.219 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2290732333' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-22T15:05:51.219 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3425636982' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-22T15:05:51.219 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: Activating manager daemon y 2024-04-22T15:05:51.220 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mgrmap e2: y(active, starting, since 0.00285119s) 2024-04-22T15:05:51.220 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14100 172.21.15.145:0/1934174902' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:05:51.220 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14100 172.21.15.145:0/1934174902' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:05:51.220 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14100 172.21.15.145:0/1934174902' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:05:51.220 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14100 172.21.15.145:0/1934174902' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:05:51.220 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14100 172.21.15.145:0/1934174902' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T15:05:51.220 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: Manager daemon y is now available 2024-04-22T15:05:51.220 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14100 172.21.15.145:0/1934174902' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:05:51.220 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14100 172.21.15.145:0/1934174902' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:05:51.220 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14100 172.21.15.145:0/1934174902' entity='mgr.y' 2024-04-22T15:05:51.220 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14100 172.21.15.145:0/1934174902' entity='mgr.y' 2024-04-22T15:05:51.220 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14100 172.21.15.145:0/1934174902' entity='mgr.y' 2024-04-22T15:05:51.220 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mgrmap e3: y(active, since 1.00611s) 2024-04-22T15:05:51.221 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/1907013403' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-22T15:05:51.221 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mgrmap e4: y(active, since 2s) 2024-04-22T15:05:51.221 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3016376729' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-22T15:05:51.221 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3016376729' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-22T15:05:51.221 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3681218592' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-22T15:05:51.221 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3681218592' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-22T15:05:51.221 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mgrmap e5: y(active, since 4s) 2024-04-22T15:05:51.221 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/1643849192' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-22T15:05:51.221 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: Active manager daemon y restarted 2024-04-22T15:05:51.221 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: Activating manager daemon y 2024-04-22T15:05:51.221 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: osdmap e2: 0 total, 0 up, 0 in 2024-04-22T15:05:51.221 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mgrmap e6: y(active, starting, since 0.0524228s) 2024-04-22T15:05:51.221 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:05:51.222 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T15:05:51.222 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:05:51.222 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:05:51.222 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:05:51.222 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: Manager daemon y is now available 2024-04-22T15:05:51.222 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' 2024-04-22T15:05:51.222 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' 2024-04-22T15:05:51.222 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:05:51.222 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:05:51.222 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:05:51.222 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:05:51.222 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:05:51.223 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mgrmap e7: y(active, since 1.05987s) 2024-04-22T15:05:51.223 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: [22/Apr/2024:15:04:58] ENGINE Bus STARTING 2024-04-22T15:05:51.223 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mgrmap e8: y(active, since 2s) 2024-04-22T15:05:51.223 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: [22/Apr/2024:15:04:58] ENGINE Serving on https://172.21.15.145:7150 2024-04-22T15:05:51.223 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: [22/Apr/2024:15:04:58] ENGINE Bus STARTED 2024-04-22T15:05:51.223 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' 2024-04-22T15:05:51.223 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:05:51.223 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:05:51.223 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' 2024-04-22T15:05:51.223 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:05:51.223 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:05:51.223 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:05:51.223 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: Generating ssh key... 2024-04-22T15:05:51.224 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' 2024-04-22T15:05:51.224 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' 2024-04-22T15:05:51.224 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:05:51.224 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi145", "addr": "172.21.15.145", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:05:51.224 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: Deploying cephadm binary to smithi145 2024-04-22T15:05:51.224 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' 2024-04-22T15:05:51.224 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: Added host smithi145 2024-04-22T15:05:51.224 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:05:51.224 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:05:51.224 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: Saving service mon spec with placement count:5 2024-04-22T15:05:51.224 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' 2024-04-22T15:05:51.224 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:05:51.224 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: Saving service mgr spec with placement count:2 2024-04-22T15:05:51.224 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' 2024-04-22T15:05:51.225 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' 2024-04-22T15:05:51.225 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' 2024-04-22T15:05:51.225 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3065499207' entity='client.admin' 2024-04-22T15:05:51.225 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/1891450196' entity='client.admin' 2024-04-22T15:05:51.225 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' 2024-04-22T15:05:51.225 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14118 172.21.15.145:0/3331166908' entity='mgr.y' 2024-04-22T15:05:51.225 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3748984348' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-22T15:05:51.225 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3748984348' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-22T15:05:51.225 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mgrmap e9: y(active, since 16s) 2024-04-22T15:05:51.225 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/531773053' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-22T15:05:51.225 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: Active manager daemon y restarted 2024-04-22T15:05:51.225 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: Activating manager daemon y 2024-04-22T15:05:51.225 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: osdmap e3: 0 total, 0 up, 0 in 2024-04-22T15:05:51.226 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mgrmap e10: y(active, starting, since 0.650516s) 2024-04-22T15:05:51.226 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:05:51.226 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T15:05:51.226 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:05:51.226 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:05:51.226 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:05:51.226 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: Manager daemon y is now available 2024-04-22T15:05:51.226 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:05:51.226 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:05:51.226 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:05:51.226 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:05:51.226 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mgrmap e11: y(active, since 1.65296s) 2024-04-22T15:05:51.227 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: [22/Apr/2024:15:05:20] ENGINE Bus STARTING 2024-04-22T15:05:51.227 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: [22/Apr/2024:15:05:21] ENGINE Serving on https://172.21.15.145:7150 2024-04-22T15:05:51.227 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: [22/Apr/2024:15:05:21] ENGINE Bus STARTED 2024-04-22T15:05:51.227 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:51.227 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:51.227 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mgrmap e12: y(active, since 2s) 2024-04-22T15:05:51.227 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:51.227 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:51.227 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:51.227 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:05:51.227 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:51.227 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:05:51.227 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/1784484035' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-22T15:05:51.228 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mgrmap e13: y(active, since 6s) 2024-04-22T15:05:51.228 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2882001211' entity='client.admin' 2024-04-22T15:05:51.228 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/537971518' entity='client.admin' 2024-04-22T15:05:51.228 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:51.228 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi145", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:05:51.228 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:51.228 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:05:51.228 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:51.228 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:05:51.228 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:51.228 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:05:51.228 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:05:51.228 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:05:51.229 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: Updating smithi145:/etc/ceph/ceph.conf 2024-04-22T15:05:51.229 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:05:51.229 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:51.229 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:51.229 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi172", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:05:51.229 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: Deploying cephadm binary to smithi172 2024-04-22T15:05:51.229 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:51.229 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: Added host smithi172 2024-04-22T15:05:51.229 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:05:51.229 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:05:51.229 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:05:51.229 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:51.229 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:51.230 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:51.230 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:51.230 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T15:05:51.230 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:51.230 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:51.230 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:51.230 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi172", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:05:51.230 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:51.230 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:51.230 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3407015052' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-22T15:05:51.230 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3407015052' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-22T15:05:51.230 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: osdmap e4: 0 total, 0 up, 0 in 2024-04-22T15:05:51.231 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:51.231 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi145:172.21.15.145=a;smithi145:[v2:172.21.15.145:3301,v1:172.21.15.145:6790]=c;smithi172:172.21.15.172=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:05:51.231 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: Saving service mon spec with placement smithi145:172.21.15.145=a;smithi145:[v2:172.21.15.145:3301,v1:172.21.15.145:6790]=c;smithi172:172.21.15.172=b;count:3 2024-04-22T15:05:51.231 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:51.231 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:05:51.231 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:05:51.231 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:05:51.231 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: Updating smithi172:/etc/ceph/ceph.conf 2024-04-22T15:05:51.231 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:05:51.231 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:51.231 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:51.231 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:05:51.232 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:05:51.232 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: Deploying daemon mon.c on smithi145 2024-04-22T15:05:51.232 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:51.232 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:51.232 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: from='client.? 172.21.15.172:0/3177305875' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T15:05:51.232 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-04-22T15:05:51.232 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 ceph-mon[26606]: mon.c@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-04-22T15:05:51.232 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 bash[26588]: b71d9b9f850909b4f0dbe9e34be195dea8120952106575a706c786235bbb4642 2024-04-22T15:05:51.232 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:51 smithi145 systemd[1]: Started Ceph mon.c for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:05:51.477 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-22T15:05:51.477 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T15:05:56.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:05:56.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[21899]: mon.a calling monitor election 2024-04-22T15:05:56.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:05:56.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[21899]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:56.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:05:56.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:05:56.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[21899]: mon.c calling monitor election 2024-04-22T15:05:56.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[21899]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:56.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:05:56.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:05:56.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:05:56.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[21899]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-22T15:05:56.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[21899]: monmap e2: 2 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0],c=[v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0]} 2024-04-22T15:05:56.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[21899]: fsmap 2024-04-22T15:05:56.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[21899]: osdmap e4: 0 total, 0 up, 0 in 2024-04-22T15:05:56.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[21899]: mgrmap e13: y(active, since 36s) 2024-04-22T15:05:56.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[21899]: overall HEALTH_OK 2024-04-22T15:05:56.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:56.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:05:56.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:05:56.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:05:56.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[26606]: mon.a calling monitor election 2024-04-22T15:05:56.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:05:56.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[26606]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:56.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:05:56.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:05:56.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[26606]: mon.c calling monitor election 2024-04-22T15:05:56.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[26606]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:56.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:05:56.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:05:56.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:05:56.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[26606]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-22T15:05:56.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[26606]: monmap e2: 2 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0],c=[v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0]} 2024-04-22T15:05:56.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[26606]: fsmap 2024-04-22T15:05:56.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[26606]: osdmap e4: 0 total, 0 up, 0 in 2024-04-22T15:05:56.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[26606]: mgrmap e13: y(active, since 36s) 2024-04-22T15:05:56.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[26606]: overall HEALTH_OK 2024-04-22T15:05:56.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:05:56.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:05:56.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:56 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:05:56.467 INFO:teuthology.orchestra.run.smithi172.stdout: 2024-04-22T15:05:56.467 INFO:teuthology.orchestra.run.smithi172.stdout:{"epoch":2,"fsid":"869afcfa-00b9-11ef-bc93-c7b262605968","modified":"2024-04-22T15:05:51.103385Z","created":"2024-04-22T15:04:30.924274Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:3300","nonce":0},{"type":"v1","addr":"172.21.15.145:6789","nonce":0}]},"addr":"172.21.15.145:6789/0","public_addr":"172.21.15.145:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:3301","nonce":0},{"type":"v1","addr":"172.21.15.145:6790","nonce":0}]},"addr":"172.21.15.145:6790/0","public_addr":"172.21.15.145:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-22T15:05:56.469 INFO:teuthology.orchestra.run.smithi172.stderr:dumped monmap epoch 2 2024-04-22T15:05:57.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:57 smithi145 ceph-mon[21899]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:57.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:57 smithi145 ceph-mon[21899]: Deploying daemon mon.b on smithi172 2024-04-22T15:05:57.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:57 smithi145 ceph-mon[21899]: from='client.? 172.21.15.172:0/1698800283' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T15:05:57.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:57 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:05:57.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:57 smithi145 ceph-mon[26606]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:57.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:57 smithi145 ceph-mon[26606]: Deploying daemon mon.b on smithi172 2024-04-22T15:05:57.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:57 smithi145 ceph-mon[26606]: from='client.? 172.21.15.172:0/1698800283' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T15:05:57.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:57 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:05:58.600 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-22T15:05:58.601 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T15:05:58.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:05:58 smithi145 ceph-mon[21899]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:05:58.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:05:58 smithi145 ceph-mon[26606]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:00.806 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:00 smithi172 podman[22713]: 2024-04-22 15:06:00.692078267 +0000 UTC m=+0.018612389 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-22T15:06:00.807 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:00 smithi172 podman[22713]: 2024-04-22 15:06:00.802609653 +0000 UTC m=+0.129143757 container create a2fc3297953a9de93e007b77fdd15db0bfd523c2a04fe6c7a0cc6f352f167da4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-b, com.redhat.component=centos-stream-container, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, architecture=x86_64, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, io.openshift.expose-services=, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, name=centos-stream, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , version=8) 2024-04-22T15:06:00.912 INFO:teuthology.orchestra.run.smithi172.stdout: 2024-04-22T15:06:00.912 INFO:teuthology.orchestra.run.smithi172.stdout:{"epoch":2,"fsid":"869afcfa-00b9-11ef-bc93-c7b262605968","modified":"2024-04-22T15:05:51.103385Z","created":"2024-04-22T15:04:30.924274Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:3300","nonce":0},{"type":"v1","addr":"172.21.15.145:6789","nonce":0}]},"addr":"172.21.15.145:6789/0","public_addr":"172.21.15.145:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:3301","nonce":0},{"type":"v1","addr":"172.21.15.145:6790","nonce":0}]},"addr":"172.21.15.145:6790/0","public_addr":"172.21.15.145:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-22T15:06:00.914 INFO:teuthology.orchestra.run.smithi172.stderr:dumped monmap epoch 2 2024-04-22T15:06:01.061 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 podman[22713]: 2024-04-22 15:06:01.013714236 +0000 UTC m=+0.340248343 container init a2fc3297953a9de93e007b77fdd15db0bfd523c2a04fe6c7a0cc6f352f167da4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-b, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, name=centos-stream, ceph=True, release=754, vendor=Red Hat, Inc., vcs-type=git, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD) 2024-04-22T15:06:01.061 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 podman[22713]: 2024-04-22 15:06:01.016962385 +0000 UTC m=+0.343496481 container start a2fc3297953a9de93e007b77fdd15db0bfd523c2a04fe6c7a0cc6f352f167da4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-b, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, io.openshift.expose-services=, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, ceph=True, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git) 2024-04-22T15:06:01.061 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: set uid:gid to 167:167 (ceph:ceph) 2024-04-22T15:06:01.061 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-22T15:06:01.061 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: pidfile_write: ignore empty --pid-file 2024-04-22T15:06:01.061 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: load: jerasure load: lrc 2024-04-22T15:06:01.062 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: RocksDB version: 6.15.5 2024-04-22T15:06:01.062 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-22T15:06:01.062 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Compile date Apr 18 2022 2024-04-22T15:06:01.062 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: DB SUMMARY 2024-04-22T15:06:01.062 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: DB Session ID: SE4O8CQCI9TQ1ZK80EQ1 2024-04-22T15:06:01.062 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: CURRENT file: CURRENT 2024-04-22T15:06:01.062 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: IDENTITY file: IDENTITY 2024-04-22T15:06:01.062 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-04-22T15:06:01.062 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-04-22T15:06:01.062 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 636 ; 2024-04-22T15:06:01.062 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.error_if_exists: 0 2024-04-22T15:06:01.062 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.create_if_missing: 0 2024-04-22T15:06:01.062 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.paranoid_checks: 1 2024-04-22T15:06:01.062 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-22T15:06:01.062 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.env: 0x56016a6c6860 2024-04-22T15:06:01.063 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.fs: Posix File System 2024-04-22T15:06:01.063 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.info_log: 0x56016b461dc0 2024-04-22T15:06:01.063 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.max_file_opening_threads: 16 2024-04-22T15:06:01.063 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.statistics: (nil) 2024-04-22T15:06:01.063 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.use_fsync: 0 2024-04-22T15:06:01.063 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.max_log_file_size: 0 2024-04-22T15:06:01.063 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-22T15:06:01.063 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-22T15:06:01.063 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.keep_log_file_num: 1000 2024-04-22T15:06:01.063 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.recycle_log_file_num: 0 2024-04-22T15:06:01.063 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.allow_fallocate: 1 2024-04-22T15:06:01.063 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.allow_mmap_reads: 0 2024-04-22T15:06:01.063 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.allow_mmap_writes: 0 2024-04-22T15:06:01.063 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.use_direct_reads: 0 2024-04-22T15:06:01.064 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-22T15:06:01.064 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.create_missing_column_families: 0 2024-04-22T15:06:01.064 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.db_log_dir: 2024-04-22T15:06:01.064 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2024-04-22T15:06:01.064 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-22T15:06:01.064 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-22T15:06:01.064 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-22T15:06:01.064 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-22T15:06:01.064 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-22T15:06:01.064 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-22T15:06:01.065 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.advise_random_on_open: 1 2024-04-22T15:06:01.065 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.db_write_buffer_size: 0 2024-04-22T15:06:01.066 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.write_buffer_manager: 0x56016b552240 2024-04-22T15:06:01.066 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-22T15:06:01.066 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-22T15:06:01.066 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-22T15:06:01.066 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-22T15:06:01.066 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.rate_limiter: (nil) 2024-04-22T15:06:01.066 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-22T15:06:01.066 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.wal_recovery_mode: 2 2024-04-22T15:06:01.066 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.enable_thread_tracking: 0 2024-04-22T15:06:01.066 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.enable_pipelined_write: 0 2024-04-22T15:06:01.066 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.unordered_write: 0 2024-04-22T15:06:01.066 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-22T15:06:01.066 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-22T15:06:01.066 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-22T15:06:01.067 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-22T15:06:01.067 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.row_cache: None 2024-04-22T15:06:01.067 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.wal_filter: None 2024-04-22T15:06:01.067 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-22T15:06:01.067 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.allow_ingest_behind: 0 2024-04-22T15:06:01.067 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.preserve_deletes: 0 2024-04-22T15:06:01.067 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.two_write_queues: 0 2024-04-22T15:06:01.068 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.manual_wal_flush: 0 2024-04-22T15:06:01.068 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.atomic_flush: 0 2024-04-22T15:06:01.068 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-22T15:06:01.068 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-22T15:06:01.069 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-22T15:06:01.069 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.log_readahead_size: 0 2024-04-22T15:06:01.069 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-22T15:06:01.069 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.best_efforts_recovery: 0 2024-04-22T15:06:01.069 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-22T15:06:01.069 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-22T15:06:01.069 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.allow_data_in_errors: 0 2024-04-22T15:06:01.069 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.db_host_id: __hostname__ 2024-04-22T15:06:01.069 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.max_background_jobs: 2 2024-04-22T15:06:01.069 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.max_background_compactions: -1 2024-04-22T15:06:01.069 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.max_subcompactions: 1 2024-04-22T15:06:01.069 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-22T15:06:01.069 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-22T15:06:01.069 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-22T15:06:01.069 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.max_total_wal_size: 0 2024-04-22T15:06:01.070 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-22T15:06:01.070 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-22T15:06:01.070 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-22T15:06:01.070 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-22T15:06:01.070 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.max_open_files: -1 2024-04-22T15:06:01.070 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.bytes_per_sync: 0 2024-04-22T15:06:01.070 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-22T15:06:01.070 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-22T15:06:01.070 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.compaction_readahead_size: 0 2024-04-22T15:06:01.070 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.max_background_flushes: -1 2024-04-22T15:06:01.070 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Compression algorithms supported: 2024-04-22T15:06:01.070 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-22T15:06:01.070 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: kZSTD supported: 0 2024-04-22T15:06:01.070 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: kXpressCompression supported: 0 2024-04-22T15:06:01.071 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: kLZ4HCCompression supported: 1 2024-04-22T15:06:01.071 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: kLZ4Compression supported: 1 2024-04-22T15:06:01.071 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: kBZip2Compression supported: 0 2024-04-22T15:06:01.071 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: kZlibCompression supported: 1 2024-04-22T15:06:01.071 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: kSnappyCompression supported: 1 2024-04-22T15:06:01.071 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-22T15:06:01.071 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2024-04-22T15:06:01.071 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-22T15:06:01.071 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-22T15:06:01.071 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.merge_operator: 2024-04-22T15:06:01.071 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.compaction_filter: None 2024-04-22T15:06:01.071 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.compaction_filter_factory: None 2024-04-22T15:06:01.071 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.sst_partitioner_factory: None 2024-04-22T15:06:01.071 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-22T15:06:01.071 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-22T15:06:01.071 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56016b42dd00) 2024-04-22T15:06:01.072 INFO:journalctl@ceph.mon.b.smithi172.stdout: cache_index_and_filter_blocks: 1 2024-04-22T15:06:01.072 INFO:journalctl@ceph.mon.b.smithi172.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-22T15:06:01.072 INFO:journalctl@ceph.mon.b.smithi172.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-22T15:06:01.072 INFO:journalctl@ceph.mon.b.smithi172.stdout: pin_top_level_index_and_filter: 1 2024-04-22T15:06:01.072 INFO:journalctl@ceph.mon.b.smithi172.stdout: index_type: 0 2024-04-22T15:06:01.072 INFO:journalctl@ceph.mon.b.smithi172.stdout: data_block_index_type: 0 2024-04-22T15:06:01.072 INFO:journalctl@ceph.mon.b.smithi172.stdout: index_shortening: 1 2024-04-22T15:06:01.072 INFO:journalctl@ceph.mon.b.smithi172.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-22T15:06:01.072 INFO:journalctl@ceph.mon.b.smithi172.stdout: hash_index_allow_collision: 1 2024-04-22T15:06:01.072 INFO:journalctl@ceph.mon.b.smithi172.stdout: checksum: 1 2024-04-22T15:06:01.072 INFO:journalctl@ceph.mon.b.smithi172.stdout: no_block_cache: 0 2024-04-22T15:06:01.072 INFO:journalctl@ceph.mon.b.smithi172.stdout: block_cache: 0x56016b498170 2024-04-22T15:06:01.072 INFO:journalctl@ceph.mon.b.smithi172.stdout: block_cache_name: BinnedLRUCache 2024-04-22T15:06:01.072 INFO:journalctl@ceph.mon.b.smithi172.stdout: block_cache_options: 2024-04-22T15:06:01.072 INFO:journalctl@ceph.mon.b.smithi172.stdout: capacity : 536870912 2024-04-22T15:06:01.073 INFO:journalctl@ceph.mon.b.smithi172.stdout: num_shard_bits : 4 2024-04-22T15:06:01.073 INFO:journalctl@ceph.mon.b.smithi172.stdout: strict_capacity_limit : 0 2024-04-22T15:06:01.073 INFO:journalctl@ceph.mon.b.smithi172.stdout: high_pri_pool_ratio: 0.000 2024-04-22T15:06:01.073 INFO:journalctl@ceph.mon.b.smithi172.stdout: block_cache_compressed: (nil) 2024-04-22T15:06:01.073 INFO:journalctl@ceph.mon.b.smithi172.stdout: persistent_cache: (nil) 2024-04-22T15:06:01.073 INFO:journalctl@ceph.mon.b.smithi172.stdout: block_size: 4096 2024-04-22T15:06:01.073 INFO:journalctl@ceph.mon.b.smithi172.stdout: block_size_deviation: 10 2024-04-22T15:06:01.073 INFO:journalctl@ceph.mon.b.smithi172.stdout: block_restart_interval: 16 2024-04-22T15:06:01.073 INFO:journalctl@ceph.mon.b.smithi172.stdout: index_block_restart_interval: 1 2024-04-22T15:06:01.073 INFO:journalctl@ceph.mon.b.smithi172.stdout: metadata_block_size: 4096 2024-04-22T15:06:01.073 INFO:journalctl@ceph.mon.b.smithi172.stdout: partition_filters: 0 2024-04-22T15:06:01.073 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_delta_encoding: 1 2024-04-22T15:06:01.073 INFO:journalctl@ceph.mon.b.smithi172.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-22T15:06:01.073 INFO:journalctl@ceph.mon.b.smithi172.stdout: whole_key_filtering: 1 2024-04-22T15:06:01.073 INFO:journalctl@ceph.mon.b.smithi172.stdout: verify_compression: 0 2024-04-22T15:06:01.073 INFO:journalctl@ceph.mon.b.smithi172.stdout: read_amp_bytes_per_bit: 0 2024-04-22T15:06:01.074 INFO:journalctl@ceph.mon.b.smithi172.stdout: format_version: 4 2024-04-22T15:06:01.074 INFO:journalctl@ceph.mon.b.smithi172.stdout: enable_index_compression: 1 2024-04-22T15:06:01.074 INFO:journalctl@ceph.mon.b.smithi172.stdout: block_align: 0 2024-04-22T15:06:01.074 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.write_buffer_size: 33554432 2024-04-22T15:06:01.074 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.max_write_buffer_number: 2 2024-04-22T15:06:01.074 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.compression: NoCompression 2024-04-22T15:06:01.074 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.bottommost_compression: Disabled 2024-04-22T15:06:01.074 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.prefix_extractor: nullptr 2024-04-22T15:06:01.074 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-22T15:06:01.074 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.num_levels: 7 2024-04-22T15:06:01.074 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-22T15:06:01.074 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-22T15:06:01.074 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-22T15:06:01.074 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-22T15:06:01.074 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-22T15:06:01.075 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-22T15:06:01.075 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-22T15:06:01.075 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-22T15:06:01.075 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-22T15:06:01.075 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-22T15:06:01.075 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-22T15:06:01.075 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.compression_opts.level: 32767 2024-04-22T15:06:01.075 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.compression_opts.strategy: 0 2024-04-22T15:06:01.075 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-22T15:06:01.075 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-22T15:06:01.075 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-22T15:06:01.075 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.compression_opts.enabled: false 2024-04-22T15:06:01.075 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-22T15:06:01.075 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-22T15:06:01.075 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-22T15:06:01.075 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.target_file_size_base: 67108864 2024-04-22T15:06:01.076 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-22T15:06:01.076 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-22T15:06:01.076 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-22T15:06:01.076 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-22T15:06:01.076 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-22T15:06:01.076 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-22T15:06:01.076 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-22T15:06:01.076 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-22T15:06:01.076 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-22T15:06:01.076 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-22T15:06:01.076 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-22T15:06:01.077 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-22T15:06:01.077 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-22T15:06:01.077 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.arena_block_size: 4194304 2024-04-22T15:06:01.077 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-22T15:06:01.077 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-22T15:06:01.077 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-22T15:06:01.077 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.disable_auto_compactions: 0 2024-04-22T15:06:01.077 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-22T15:06:01.077 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-22T15:06:01.077 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-22T15:06:01.077 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-22T15:06:01.077 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-22T15:06:01.077 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-22T15:06:01.077 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-22T15:06:01.077 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-22T15:06:01.078 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-22T15:06:01.078 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-22T15:06:01.078 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.table_properties_collectors: 2024-04-22T15:06:01.078 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.inplace_update_support: 0 2024-04-22T15:06:01.078 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-22T15:06:01.078 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-22T15:06:01.078 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-22T15:06:01.079 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-22T15:06:01.079 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.bloom_locality: 0 2024-04-22T15:06:01.079 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.max_successive_merges: 0 2024-04-22T15:06:01.079 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-22T15:06:01.079 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.paranoid_file_checks: 0 2024-04-22T15:06:01.079 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.force_consistency_checks: 1 2024-04-22T15:06:01.079 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.report_bg_io_stats: 0 2024-04-22T15:06:01.079 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.ttl: 2592000 2024-04-22T15:06:01.079 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-22T15:06:01.079 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.enable_blob_files: false 2024-04-22T15:06:01.080 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.min_blob_size: 0 2024-04-22T15:06:01.080 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.blob_file_size: 268435456 2024-04-22T15:06:01.080 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-22T15:06:01.080 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-22T15:06:01.080 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-22T15:06:01.080 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-22T15:06:01.080 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-04-22T15:06:01.080 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-04-22T15:06:01.080 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713798361043032, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-22T15:06:01.080 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-04-22T15:06:01.080 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-22T15:06:01.080 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713798361043340, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1713798361, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "648dc050-47c2-4716-adff-c028a50c0c24", "db_session_id": "SE4O8CQCI9TQ1ZK80EQ1"}} 2024-04-22T15:06:01.080 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-04-22T15:06:01.080 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713798361043652, "job": 1, "event": "recovery_finished"} 2024-04-22T15:06:01.080 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-22T15:06:01.081 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x56016b47ea80 2024-04-22T15:06:01.081 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: DB pointer 0x56016b48e000 2024-04-22T15:06:01.081 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-22T15:06:01.081 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-22T15:06:01.081 INFO:journalctl@ceph.mon.b.smithi172.stdout: ** DB Stats ** 2024-04-22T15:06:01.081 INFO:journalctl@ceph.mon.b.smithi172.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T15:06:01.081 INFO:journalctl@ceph.mon.b.smithi172.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-22T15:06:01.081 INFO:journalctl@ceph.mon.b.smithi172.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-22T15:06:01.081 INFO:journalctl@ceph.mon.b.smithi172.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T15:06:01.081 INFO:journalctl@ceph.mon.b.smithi172.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-22T15:06:01.081 INFO:journalctl@ceph.mon.b.smithi172.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-22T15:06:01.081 INFO:journalctl@ceph.mon.b.smithi172.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T15:06:01.081 INFO:journalctl@ceph.mon.b.smithi172.stdout: 2024-04-22T15:06:01.081 INFO:journalctl@ceph.mon.b.smithi172.stdout: ** Compaction Stats [default] ** 2024-04-22T15:06:01.081 INFO:journalctl@ceph.mon.b.smithi172.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T15:06:01.082 INFO:journalctl@ceph.mon.b.smithi172.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T15:06:01.082 INFO:journalctl@ceph.mon.b.smithi172.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-22T15:06:01.082 INFO:journalctl@ceph.mon.b.smithi172.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-22T15:06:01.082 INFO:journalctl@ceph.mon.b.smithi172.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-22T15:06:01.082 INFO:journalctl@ceph.mon.b.smithi172.stdout: 2024-04-22T15:06:01.082 INFO:journalctl@ceph.mon.b.smithi172.stdout: ** Compaction Stats [default] ** 2024-04-22T15:06:01.082 INFO:journalctl@ceph.mon.b.smithi172.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T15:06:01.082 INFO:journalctl@ceph.mon.b.smithi172.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T15:06:01.082 INFO:journalctl@ceph.mon.b.smithi172.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-22T15:06:01.082 INFO:journalctl@ceph.mon.b.smithi172.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T15:06:01.083 INFO:journalctl@ceph.mon.b.smithi172.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-22T15:06:01.083 INFO:journalctl@ceph.mon.b.smithi172.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-22T15:06:01.083 INFO:journalctl@ceph.mon.b.smithi172.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-22T15:06:01.083 INFO:journalctl@ceph.mon.b.smithi172.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-22T15:06:01.083 INFO:journalctl@ceph.mon.b.smithi172.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-22T15:06:01.083 INFO:journalctl@ceph.mon.b.smithi172.stdout: Cumulative compaction: 0.00 GB write, 0.64 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T15:06:01.083 INFO:journalctl@ceph.mon.b.smithi172.stdout: Interval compaction: 0.00 GB write, 0.64 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T15:06:01.083 INFO:journalctl@ceph.mon.b.smithi172.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-22T15:06:01.083 INFO:journalctl@ceph.mon.b.smithi172.stdout: 2024-04-22T15:06:01.083 INFO:journalctl@ceph.mon.b.smithi172.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-22T15:06:01.083 INFO:journalctl@ceph.mon.b.smithi172.stdout: 2024-04-22T15:06:01.083 INFO:journalctl@ceph.mon.b.smithi172.stdout: ** Compaction Stats [default] ** 2024-04-22T15:06:01.083 INFO:journalctl@ceph.mon.b.smithi172.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T15:06:01.083 INFO:journalctl@ceph.mon.b.smithi172.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T15:06:01.084 INFO:journalctl@ceph.mon.b.smithi172.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-22T15:06:01.084 INFO:journalctl@ceph.mon.b.smithi172.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-22T15:06:01.084 INFO:journalctl@ceph.mon.b.smithi172.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-22T15:06:01.084 INFO:journalctl@ceph.mon.b.smithi172.stdout: 2024-04-22T15:06:01.084 INFO:journalctl@ceph.mon.b.smithi172.stdout: ** Compaction Stats [default] ** 2024-04-22T15:06:01.084 INFO:journalctl@ceph.mon.b.smithi172.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T15:06:01.084 INFO:journalctl@ceph.mon.b.smithi172.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T15:06:01.084 INFO:journalctl@ceph.mon.b.smithi172.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-22T15:06:01.084 INFO:journalctl@ceph.mon.b.smithi172.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T15:06:01.084 INFO:journalctl@ceph.mon.b.smithi172.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-22T15:06:01.084 INFO:journalctl@ceph.mon.b.smithi172.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-22T15:06:01.084 INFO:journalctl@ceph.mon.b.smithi172.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-22T15:06:01.084 INFO:journalctl@ceph.mon.b.smithi172.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-22T15:06:01.084 INFO:journalctl@ceph.mon.b.smithi172.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-22T15:06:01.084 INFO:journalctl@ceph.mon.b.smithi172.stdout: Cumulative compaction: 0.00 GB write, 0.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T15:06:01.085 INFO:journalctl@ceph.mon.b.smithi172.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T15:06:01.085 INFO:journalctl@ceph.mon.b.smithi172.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-22T15:06:01.085 INFO:journalctl@ceph.mon.b.smithi172.stdout: 2024-04-22T15:06:01.085 INFO:journalctl@ceph.mon.b.smithi172.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-22T15:06:01.085 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-04-22T15:06:01.085 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: using public_addr v2:172.21.15.172:0/0 -> [v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0] 2024-04-22T15:06:01.085 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: starting mon.b rank -1 at public addrs [v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0] at bind addrs [v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:06:01.085 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: mon.b@-1(???) e0 preinit fsid 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:06:01.085 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: mon.b@-1(synchronizing).mds e1 new map 2024-04-22T15:06:01.085 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: mon.b@-1(synchronizing).mds e1 print_map 2024-04-22T15:06:01.085 INFO:journalctl@ceph.mon.b.smithi172.stdout: e1 2024-04-22T15:06:01.085 INFO:journalctl@ceph.mon.b.smithi172.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-22T15:06:01.085 INFO:journalctl@ceph.mon.b.smithi172.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-22T15:06:01.085 INFO:journalctl@ceph.mon.b.smithi172.stdout: legacy client fscid: -1 2024-04-22T15:06:01.085 INFO:journalctl@ceph.mon.b.smithi172.stdout: 2024-04-22T15:06:01.086 INFO:journalctl@ceph.mon.b.smithi172.stdout: No filesystems configured 2024-04-22T15:06:01.086 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-22T15:06:01.086 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-22T15:06:01.086 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-22T15:06:01.086 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-22T15:06:01.086 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-22T15:06:01.086 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-22T15:06:01.086 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-22T15:06:01.086 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T15:06:01.086 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T15:06:01.086 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T15:06:01.086 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:01.086 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: Deploying daemon mon.b on smithi172 2024-04-22T15:06:01.086 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: from='client.? 172.21.15.172:0/1698800283' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T15:06:01.086 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:06:01.087 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:01.087 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:01.087 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: from='client.? 172.21.15.172:0/623089201' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T15:06:01.087 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-04-22T15:06:01.087 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 ceph-mon[22747]: mon.b@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-04-22T15:06:01.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:00 smithi145 ceph-mon[21899]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:01.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:00 smithi145 ceph-mon[21899]: from='client.? 172.21.15.172:0/623089201' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T15:06:01.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:00 smithi145 ceph-mon[26606]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:01.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:00 smithi145 ceph-mon[26606]: from='client.? 172.21.15.172:0/623089201' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T15:06:01.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 bash[22713]: a2fc3297953a9de93e007b77fdd15db0bfd523c2a04fe6c7a0cc6f352f167da4 2024-04-22T15:06:01.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:01 smithi172 systemd[1]: Started Ceph mon.b for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:06:02.700 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-22T15:06:02.700 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T15:06:06.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:06:06.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[26606]: mon.a calling monitor election 2024-04-22T15:06:06.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[26606]: mon.c calling monitor election 2024-04-22T15:06:06.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:06:06.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:06:06.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[26606]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:06.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:06:06.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:06:06.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[26606]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:06.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:06:06.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:06:06.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:06:06.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[26606]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-22T15:06:06.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[26606]: monmap e3: 3 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0],b=[v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0],c=[v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0]} 2024-04-22T15:06:06.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[26606]: fsmap 2024-04-22T15:06:06.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[26606]: osdmap e4: 0 total, 0 up, 0 in 2024-04-22T15:06:06.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[26606]: mgrmap e13: y(active, since 46s) 2024-04-22T15:06:06.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[26606]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-04-22T15:06:06.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[26606]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-04-22T15:06:06.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[26606]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-04-22T15:06:06.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[26606]: mon.b (rank 2) addr [v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0] is down (out of quorum) 2024-04-22T15:06:06.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:06.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:06:06.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:06.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:06:06.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:06:06.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[21899]: mon.a calling monitor election 2024-04-22T15:06:06.377 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[21899]: mon.c calling monitor election 2024-04-22T15:06:06.377 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:06:06.377 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:06:06.377 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[21899]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:06.377 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:06:06.377 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:06:06.377 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[21899]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:06.377 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:06:06.377 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:06:06.377 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:06:06.378 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[21899]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-22T15:06:06.378 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[21899]: monmap e3: 3 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0],b=[v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0],c=[v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0]} 2024-04-22T15:06:06.378 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[21899]: fsmap 2024-04-22T15:06:06.378 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[21899]: osdmap e4: 0 total, 0 up, 0 in 2024-04-22T15:06:06.378 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[21899]: mgrmap e13: y(active, since 46s) 2024-04-22T15:06:06.378 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[21899]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-04-22T15:06:06.378 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[21899]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-04-22T15:06:06.378 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[21899]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-04-22T15:06:06.378 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[21899]: mon.b (rank 2) addr [v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0] is down (out of quorum) 2024-04-22T15:06:06.378 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:06.379 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:06:06.379 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:06.379 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:06 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:06:07.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:07 smithi145 ceph-mon[21899]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:07.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:07 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:07.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:07 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:06:07.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:07 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:07.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:07 smithi145 ceph-mon[26606]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:07.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:07 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:07.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:07 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:06:07.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:07 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:07.536 INFO:teuthology.orchestra.run.smithi172.stdout: 2024-04-22T15:06:07.536 INFO:teuthology.orchestra.run.smithi172.stdout:{"epoch":3,"fsid":"869afcfa-00b9-11ef-bc93-c7b262605968","modified":"2024-04-22T15:06:01.062345Z","created":"2024-04-22T15:04:30.924274Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:3300","nonce":0},{"type":"v1","addr":"172.21.15.145:6789","nonce":0}]},"addr":"172.21.15.145:6789/0","public_addr":"172.21.15.145:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:3301","nonce":0},{"type":"v1","addr":"172.21.15.145:6790","nonce":0}]},"addr":"172.21.15.145:6790/0","public_addr":"172.21.15.145:6790/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:3300","nonce":0},{"type":"v1","addr":"172.21.15.172:6789","nonce":0}]},"addr":"172.21.15.172:6789/0","public_addr":"172.21.15.172:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-22T15:06:07.537 INFO:teuthology.orchestra.run.smithi172.stderr:dumped monmap epoch 3 2024-04-22T15:06:08.098 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:06:08.098 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: mon.a calling monitor election 2024-04-22T15:06:08.099 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: mon.c calling monitor election 2024-04-22T15:06:08.099 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:06:08.099 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:06:08.099 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:08.099 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:06:08.099 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:06:08.099 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:08.099 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:06:08.099 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:06:08.099 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:06:08.099 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-22T15:06:08.099 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: monmap e3: 3 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0],b=[v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0],c=[v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0]} 2024-04-22T15:06:08.099 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: fsmap 2024-04-22T15:06:08.099 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: osdmap e4: 0 total, 0 up, 0 in 2024-04-22T15:06:08.100 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: mgrmap e13: y(active, since 46s) 2024-04-22T15:06:08.100 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-04-22T15:06:08.100 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-04-22T15:06:08.100 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-04-22T15:06:08.100 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: mon.b (rank 2) addr [v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0] is down (out of quorum) 2024-04-22T15:06:08.100 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:08.100 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:06:08.100 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:08.100 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:06:08.100 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:08.100 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:08.100 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:06:08.100 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:08.120 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-22T15:06:08.120 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph config generate-minimal-conf 2024-04-22T15:06:08.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: mon.b calling monitor election 2024-04-22T15:06:08.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: Reconfiguring mon.a (unknown last config time)... 2024-04-22T15:06:08.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: Reconfiguring daemon mon.a on smithi145 2024-04-22T15:06:08.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:08.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: mon.b calling monitor election 2024-04-22T15:06:08.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: mon.c calling monitor election 2024-04-22T15:06:08.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: mon.a calling monitor election 2024-04-22T15:06:08.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-22T15:06:08.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: monmap e3: 3 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0],b=[v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0],c=[v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0]} 2024-04-22T15:06:08.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: fsmap 2024-04-22T15:06:08.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: osdmap e4: 0 total, 0 up, 0 in 2024-04-22T15:06:08.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: mgrmap e13: y(active, since 48s) 2024-04-22T15:06:08.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-04-22T15:06:08.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: Cluster is now healthy 2024-04-22T15:06:08.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:08 smithi172 ceph-mon[22747]: overall HEALTH_OK 2024-04-22T15:06:08.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:08 smithi145 ceph-mon[21899]: mon.b calling monitor election 2024-04-22T15:06:08.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:08 smithi145 ceph-mon[21899]: Reconfiguring mon.a (unknown last config time)... 2024-04-22T15:06:08.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:08 smithi145 ceph-mon[21899]: Reconfiguring daemon mon.a on smithi145 2024-04-22T15:06:08.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:08 smithi145 ceph-mon[21899]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:08.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:08 smithi145 ceph-mon[21899]: mon.b calling monitor election 2024-04-22T15:06:08.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:08 smithi145 ceph-mon[21899]: mon.c calling monitor election 2024-04-22T15:06:08.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:08 smithi145 ceph-mon[21899]: mon.a calling monitor election 2024-04-22T15:06:08.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:08 smithi145 ceph-mon[21899]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-22T15:06:08.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:08 smithi145 ceph-mon[21899]: monmap e3: 3 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0],b=[v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0],c=[v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0]} 2024-04-22T15:06:08.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:08 smithi145 ceph-mon[21899]: fsmap 2024-04-22T15:06:08.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:08 smithi145 ceph-mon[21899]: osdmap e4: 0 total, 0 up, 0 in 2024-04-22T15:06:08.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:08 smithi145 ceph-mon[21899]: mgrmap e13: y(active, since 48s) 2024-04-22T15:06:08.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:08 smithi145 ceph-mon[21899]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-04-22T15:06:08.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:08 smithi145 ceph-mon[21899]: Cluster is now healthy 2024-04-22T15:06:08.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:08 smithi145 ceph-mon[21899]: overall HEALTH_OK 2024-04-22T15:06:08.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:08 smithi145 ceph-mon[26606]: mon.b calling monitor election 2024-04-22T15:06:08.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:08 smithi145 ceph-mon[26606]: Reconfiguring mon.a (unknown last config time)... 2024-04-22T15:06:08.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:08 smithi145 ceph-mon[26606]: Reconfiguring daemon mon.a on smithi145 2024-04-22T15:06:08.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:08 smithi145 ceph-mon[26606]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:08.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:08 smithi145 ceph-mon[26606]: mon.b calling monitor election 2024-04-22T15:06:08.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:08 smithi145 ceph-mon[26606]: mon.c calling monitor election 2024-04-22T15:06:08.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:08 smithi145 ceph-mon[26606]: mon.a calling monitor election 2024-04-22T15:06:08.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:08 smithi145 ceph-mon[26606]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-22T15:06:08.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:08 smithi145 ceph-mon[26606]: monmap e3: 3 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0],b=[v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0],c=[v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0]} 2024-04-22T15:06:08.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:08 smithi145 ceph-mon[26606]: fsmap 2024-04-22T15:06:08.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:08 smithi145 ceph-mon[26606]: osdmap e4: 0 total, 0 up, 0 in 2024-04-22T15:06:08.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:08 smithi145 ceph-mon[26606]: mgrmap e13: y(active, since 48s) 2024-04-22T15:06:08.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:08 smithi145 ceph-mon[26606]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-04-22T15:06:08.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:08 smithi145 ceph-mon[26606]: Cluster is now healthy 2024-04-22T15:06:08.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:08 smithi145 ceph-mon[26606]: overall HEALTH_OK 2024-04-22T15:06:09.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:09 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:09.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:09 smithi145 ceph-mon[26606]: Reconfiguring mon.c (monmap changed)... 2024-04-22T15:06:09.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:09 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:06:09.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:09 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:06:09.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:09 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:09.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:09 smithi145 ceph-mon[26606]: Reconfiguring daemon mon.c on smithi145 2024-04-22T15:06:09.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:09 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:06:09.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:09 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:09.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:09 smithi145 ceph-mon[21899]: Reconfiguring mon.c (monmap changed)... 2024-04-22T15:06:09.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:09 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:06:09.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:09 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:06:09.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:09 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:09.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:09 smithi145 ceph-mon[21899]: Reconfiguring daemon mon.c on smithi145 2024-04-22T15:06:09.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:09 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:06:09.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:09 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:09.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:09 smithi172 ceph-mon[22747]: Reconfiguring mon.c (monmap changed)... 2024-04-22T15:06:09.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:09 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:06:09.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:09 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:06:09.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:09 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:09.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:09 smithi172 ceph-mon[22747]: Reconfiguring daemon mon.c on smithi145 2024-04-22T15:06:09.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:09 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:06:10.952 INFO:teuthology.orchestra.run.smithi145.stdout:# minimal ceph.conf for 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:06:10.952 INFO:teuthology.orchestra.run.smithi145.stdout:[global] 2024-04-22T15:06:10.952 INFO:teuthology.orchestra.run.smithi145.stdout: fsid = 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:06:10.952 INFO:teuthology.orchestra.run.smithi145.stdout: mon_host = [v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0] [v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0] [v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0] 2024-04-22T15:06:11.178 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:10 smithi172 ceph-mon[22747]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:11.178 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:10 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:11.182 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:10 smithi145 ceph-mon[21899]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:11.183 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:10 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:11.183 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:10 smithi145 ceph-mon[26606]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:11.183 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:10 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:11.800 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-22T15:06:11.800 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-04-22T15:06:11.801 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd of=/etc/ceph/ceph.conf 2024-04-22T15:06:11.827 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-04-22T15:06:11.827 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:06:11.892 DEBUG:teuthology.orchestra.run.smithi172:> set -ex 2024-04-22T15:06:11.892 DEBUG:teuthology.orchestra.run.smithi172:> sudo dd of=/etc/ceph/ceph.conf 2024-04-22T15:06:11.917 DEBUG:teuthology.orchestra.run.smithi172:> set -ex 2024-04-22T15:06:11.917 DEBUG:teuthology.orchestra.run.smithi172:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:06:11.981 INFO:tasks.cephadm:Adding mgr.y on smithi145 2024-04-22T15:06:11.981 INFO:tasks.cephadm:Adding mgr.x on smithi172 2024-04-22T15:06:11.981 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph orch apply mgr '2;smithi145=y;smithi172=x' 2024-04-22T15:06:12.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:11 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/3888550887' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:12.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:11 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:12.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:11 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:06:12.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:11 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:06:12.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:11 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:12.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:11 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3888550887' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:12.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:11 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:12.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:11 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:06:12.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:11 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:06:12.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:11 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:12.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:11 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3888550887' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:12.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:11 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:12.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:11 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:06:12.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:11 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:06:12.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:11 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:12.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:12 smithi172 ceph-mon[22747]: Reconfiguring mon.b (monmap changed)... 2024-04-22T15:06:12.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:12 smithi172 ceph-mon[22747]: Reconfiguring daemon mon.b on smithi172 2024-04-22T15:06:12.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:12 smithi172 ceph-mon[22747]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:12.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:12 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:12.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:12 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:06:12.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:12 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:12.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:12 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:06:12.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:12 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:13.275 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:12 smithi145 ceph-mon[21899]: Reconfiguring mon.b (monmap changed)... 2024-04-22T15:06:13.275 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:12 smithi145 ceph-mon[21899]: Reconfiguring daemon mon.b on smithi172 2024-04-22T15:06:13.275 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:12 smithi145 ceph-mon[21899]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:13.275 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:12 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:13.276 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:12 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:06:13.276 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:12 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:13.276 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:12 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:06:13.276 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:12 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:13.276 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:12 smithi145 ceph-mon[26606]: Reconfiguring mon.b (monmap changed)... 2024-04-22T15:06:13.276 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:12 smithi145 ceph-mon[26606]: Reconfiguring daemon mon.b on smithi172 2024-04-22T15:06:13.276 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:12 smithi145 ceph-mon[26606]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:13.276 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:12 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:13.276 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:12 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:06:13.276 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:12 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:13.276 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:12 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:06:13.277 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:12 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:14.016 INFO:teuthology.orchestra.run.smithi172.stdout:Scheduled mgr update... 2024-04-22T15:06:14.574 DEBUG:teuthology.orchestra.run.smithi172:mgr.x> sudo journalctl -f -n 0 -u ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mgr.x.service 2024-04-22T15:06:14.576 INFO:tasks.cephadm:Deploying OSDs... 2024-04-22T15:06:14.576 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-04-22T15:06:14.576 DEBUG:teuthology.orchestra.run.smithi145:> dd if=/scratch_devs of=/dev/stdout 2024-04-22T15:06:14.594 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-22T15:06:14.595 DEBUG:teuthology.orchestra.run.smithi145:> stat /dev/vg_nvme/lv_1 2024-04-22T15:06:14.651 INFO:teuthology.orchestra.run.smithi145.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-22T15:06:14.652 INFO:teuthology.orchestra.run.smithi145.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T15:06:14.652 INFO:teuthology.orchestra.run.smithi145.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-04-22T15:06:14.652 INFO:teuthology.orchestra.run.smithi145.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T15:06:14.652 INFO:teuthology.orchestra.run.smithi145.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T15:06:14.652 INFO:teuthology.orchestra.run.smithi145.stdout:Access: 2024-04-22 15:05:50.573625234 +0000 2024-04-22T15:06:14.652 INFO:teuthology.orchestra.run.smithi145.stdout:Modify: 2024-04-22 15:03:25.011647671 +0000 2024-04-22T15:06:14.652 INFO:teuthology.orchestra.run.smithi145.stdout:Change: 2024-04-22 15:03:25.011647671 +0000 2024-04-22T15:06:14.652 INFO:teuthology.orchestra.run.smithi145.stdout: Birth: 2024-04-22 15:03:25.011647671 +0000 2024-04-22T15:06:14.652 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-22T15:06:14.717 INFO:teuthology.orchestra.run.smithi145.stderr:1+0 records in 2024-04-22T15:06:14.717 INFO:teuthology.orchestra.run.smithi145.stderr:1+0 records out 2024-04-22T15:06:14.717 INFO:teuthology.orchestra.run.smithi145.stderr:512 bytes copied, 0.00032511 s, 1.6 MB/s 2024-04-22T15:06:14.718 DEBUG:teuthology.orchestra.run.smithi145:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-22T15:06:14.776 DEBUG:teuthology.orchestra.run.smithi145:> stat /dev/vg_nvme/lv_2 2024-04-22T15:06:14.833 INFO:teuthology.orchestra.run.smithi145.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-22T15:06:14.833 INFO:teuthology.orchestra.run.smithi145.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T15:06:14.833 INFO:teuthology.orchestra.run.smithi145.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-04-22T15:06:14.834 INFO:teuthology.orchestra.run.smithi145.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T15:06:14.834 INFO:teuthology.orchestra.run.smithi145.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T15:06:14.834 INFO:teuthology.orchestra.run.smithi145.stdout:Access: 2024-04-22 15:05:50.573625234 +0000 2024-04-22T15:06:14.834 INFO:teuthology.orchestra.run.smithi145.stdout:Modify: 2024-04-22 15:03:25.451637815 +0000 2024-04-22T15:06:14.834 INFO:teuthology.orchestra.run.smithi145.stdout:Change: 2024-04-22 15:03:25.451637815 +0000 2024-04-22T15:06:14.834 INFO:teuthology.orchestra.run.smithi145.stdout: Birth: 2024-04-22 15:03:25.451637815 +0000 2024-04-22T15:06:14.834 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-22T15:06:14.897 INFO:teuthology.orchestra.run.smithi145.stderr:1+0 records in 2024-04-22T15:06:14.897 INFO:teuthology.orchestra.run.smithi145.stderr:1+0 records out 2024-04-22T15:06:14.897 INFO:teuthology.orchestra.run.smithi145.stderr:512 bytes copied, 0.000259246 s, 2.0 MB/s 2024-04-22T15:06:14.898 DEBUG:teuthology.orchestra.run.smithi145:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-22T15:06:14.956 DEBUG:teuthology.orchestra.run.smithi145:> stat /dev/vg_nvme/lv_3 2024-04-22T15:06:15.012 INFO:teuthology.orchestra.run.smithi145.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-22T15:06:15.012 INFO:teuthology.orchestra.run.smithi145.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T15:06:15.013 INFO:teuthology.orchestra.run.smithi145.stdout:Device: 5h/5d Inode: 704 Links: 1 2024-04-22T15:06:15.013 INFO:teuthology.orchestra.run.smithi145.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T15:06:15.013 INFO:teuthology.orchestra.run.smithi145.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T15:06:15.013 INFO:teuthology.orchestra.run.smithi145.stdout:Access: 2024-04-22 15:05:50.573625234 +0000 2024-04-22T15:06:15.013 INFO:teuthology.orchestra.run.smithi145.stdout:Modify: 2024-04-22 15:03:25.898627802 +0000 2024-04-22T15:06:15.013 INFO:teuthology.orchestra.run.smithi145.stdout:Change: 2024-04-22 15:03:25.898627802 +0000 2024-04-22T15:06:15.013 INFO:teuthology.orchestra.run.smithi145.stdout: Birth: 2024-04-22 15:03:25.898627802 +0000 2024-04-22T15:06:15.013 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-22T15:06:15.077 INFO:teuthology.orchestra.run.smithi145.stderr:1+0 records in 2024-04-22T15:06:15.078 INFO:teuthology.orchestra.run.smithi145.stderr:1+0 records out 2024-04-22T15:06:15.078 INFO:teuthology.orchestra.run.smithi145.stderr:512 bytes copied, 0.00019964 s, 2.6 MB/s 2024-04-22T15:06:15.079 DEBUG:teuthology.orchestra.run.smithi145:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-22T15:06:15.135 DEBUG:teuthology.orchestra.run.smithi145:> stat /dev/vg_nvme/lv_4 2024-04-22T15:06:15.191 INFO:teuthology.orchestra.run.smithi145.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-22T15:06:15.191 INFO:teuthology.orchestra.run.smithi145.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T15:06:15.192 INFO:teuthology.orchestra.run.smithi145.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-04-22T15:06:15.192 INFO:teuthology.orchestra.run.smithi145.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T15:06:15.192 INFO:teuthology.orchestra.run.smithi145.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T15:06:15.192 INFO:teuthology.orchestra.run.smithi145.stdout:Access: 2024-04-22 15:05:50.573625234 +0000 2024-04-22T15:06:15.192 INFO:teuthology.orchestra.run.smithi145.stdout:Modify: 2024-04-22 15:03:26.284619156 +0000 2024-04-22T15:06:15.192 INFO:teuthology.orchestra.run.smithi145.stdout:Change: 2024-04-22 15:03:26.284619156 +0000 2024-04-22T15:06:15.192 INFO:teuthology.orchestra.run.smithi145.stdout: Birth: 2024-04-22 15:03:26.284619156 +0000 2024-04-22T15:06:15.192 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-22T15:06:15.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:15 smithi172 ceph-mon[22747]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:15.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:15 smithi172 ceph-mon[22747]: from='client.24110 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi145=y;smithi172=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:06:15.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:15 smithi172 ceph-mon[22747]: Saving service mgr spec with placement smithi145=y;smithi172=x;count:2 2024-04-22T15:06:15.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:15 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:15.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:15 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:06:15.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:15 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:15.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:15 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:06:15.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:15 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:15.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:15 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:06:15.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:15 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-22T15:06:15.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:15 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:06:15.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:15 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:15.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:15 smithi172 ceph-mon[22747]: Deploying daemon mgr.x on smithi172 2024-04-22T15:06:15.258 INFO:teuthology.orchestra.run.smithi145.stderr:1+0 records in 2024-04-22T15:06:15.258 INFO:teuthology.orchestra.run.smithi145.stderr:1+0 records out 2024-04-22T15:06:15.258 INFO:teuthology.orchestra.run.smithi145.stderr:512 bytes copied, 0.000180569 s, 2.8 MB/s 2024-04-22T15:06:15.260 DEBUG:teuthology.orchestra.run.smithi145:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-22T15:06:15.316 DEBUG:teuthology.orchestra.run.smithi172:> set -ex 2024-04-22T15:06:15.316 DEBUG:teuthology.orchestra.run.smithi172:> dd if=/scratch_devs of=/dev/stdout 2024-04-22T15:06:15.333 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-22T15:06:15.333 DEBUG:teuthology.orchestra.run.smithi172:> stat /dev/vg_nvme/lv_1 2024-04-22T15:06:15.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:15 smithi145 ceph-mon[21899]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:15.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:15 smithi145 ceph-mon[21899]: from='client.24110 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi145=y;smithi172=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:06:15.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:15 smithi145 ceph-mon[21899]: Saving service mgr spec with placement smithi145=y;smithi172=x;count:2 2024-04-22T15:06:15.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:15 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:15.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:15 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:06:15.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:15 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:15.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:15 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:06:15.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:15 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:15.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:15 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:06:15.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:15 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-22T15:06:15.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:15 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:06:15.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:15 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:15.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:15 smithi145 ceph-mon[21899]: Deploying daemon mgr.x on smithi172 2024-04-22T15:06:15.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:15 smithi145 ceph-mon[26606]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:15.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:15 smithi145 ceph-mon[26606]: from='client.24110 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi145=y;smithi172=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:06:15.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:15 smithi145 ceph-mon[26606]: Saving service mgr spec with placement smithi145=y;smithi172=x;count:2 2024-04-22T15:06:15.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:15 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:15.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:15 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:06:15.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:15 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:15.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:15 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:06:15.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:15 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:15.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:15 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:06:15.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:15 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-22T15:06:15.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:15 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:06:15.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:15 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:15.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:15 smithi145 ceph-mon[26606]: Deploying daemon mgr.x on smithi172 2024-04-22T15:06:15.390 INFO:teuthology.orchestra.run.smithi172.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-22T15:06:15.390 INFO:teuthology.orchestra.run.smithi172.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T15:06:15.390 INFO:teuthology.orchestra.run.smithi172.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-04-22T15:06:15.390 INFO:teuthology.orchestra.run.smithi172.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T15:06:15.390 INFO:teuthology.orchestra.run.smithi172.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T15:06:15.390 INFO:teuthology.orchestra.run.smithi172.stdout:Access: 2024-04-22 15:05:59.822048220 +0000 2024-04-22T15:06:15.390 INFO:teuthology.orchestra.run.smithi172.stdout:Modify: 2024-04-22 15:02:52.773277175 +0000 2024-04-22T15:06:15.390 INFO:teuthology.orchestra.run.smithi172.stdout:Change: 2024-04-22 15:02:52.773277175 +0000 2024-04-22T15:06:15.390 INFO:teuthology.orchestra.run.smithi172.stdout: Birth: 2024-04-22 15:02:52.773277175 +0000 2024-04-22T15:06:15.391 DEBUG:teuthology.orchestra.run.smithi172:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-22T15:06:15.453 INFO:teuthology.orchestra.run.smithi172.stderr:1+0 records in 2024-04-22T15:06:15.453 INFO:teuthology.orchestra.run.smithi172.stderr:1+0 records out 2024-04-22T15:06:15.453 INFO:teuthology.orchestra.run.smithi172.stderr:512 bytes copied, 0.000214401 s, 2.4 MB/s 2024-04-22T15:06:15.454 DEBUG:teuthology.orchestra.run.smithi172:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-22T15:06:15.510 DEBUG:teuthology.orchestra.run.smithi172:> stat /dev/vg_nvme/lv_2 2024-04-22T15:06:15.565 INFO:teuthology.orchestra.run.smithi172.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-22T15:06:15.565 INFO:teuthology.orchestra.run.smithi172.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T15:06:15.566 INFO:teuthology.orchestra.run.smithi172.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-04-22T15:06:15.566 INFO:teuthology.orchestra.run.smithi172.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T15:06:15.566 INFO:teuthology.orchestra.run.smithi172.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T15:06:15.566 INFO:teuthology.orchestra.run.smithi172.stdout:Access: 2024-04-22 15:05:59.822048220 +0000 2024-04-22T15:06:15.566 INFO:teuthology.orchestra.run.smithi172.stdout:Modify: 2024-04-22 15:02:53.230265493 +0000 2024-04-22T15:06:15.566 INFO:teuthology.orchestra.run.smithi172.stdout:Change: 2024-04-22 15:02:53.230265493 +0000 2024-04-22T15:06:15.566 INFO:teuthology.orchestra.run.smithi172.stdout: Birth: 2024-04-22 15:02:53.230265493 +0000 2024-04-22T15:06:15.566 DEBUG:teuthology.orchestra.run.smithi172:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-22T15:06:15.682 INFO:teuthology.orchestra.run.smithi172.stderr:1+0 records in 2024-04-22T15:06:15.682 INFO:teuthology.orchestra.run.smithi172.stderr:1+0 records out 2024-04-22T15:06:15.682 INFO:teuthology.orchestra.run.smithi172.stderr:512 bytes copied, 0.000192274 s, 2.7 MB/s 2024-04-22T15:06:15.683 DEBUG:teuthology.orchestra.run.smithi172:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-22T15:06:15.698 DEBUG:teuthology.orchestra.run.smithi172:> stat /dev/vg_nvme/lv_3 2024-04-22T15:06:15.757 INFO:teuthology.orchestra.run.smithi172.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-22T15:06:15.757 INFO:teuthology.orchestra.run.smithi172.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T15:06:15.757 INFO:teuthology.orchestra.run.smithi172.stdout:Device: 5h/5d Inode: 702 Links: 1 2024-04-22T15:06:15.757 INFO:teuthology.orchestra.run.smithi172.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T15:06:15.758 INFO:teuthology.orchestra.run.smithi172.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T15:06:15.758 INFO:teuthology.orchestra.run.smithi172.stdout:Access: 2024-04-22 15:06:15.636701845 +0000 2024-04-22T15:06:15.758 INFO:teuthology.orchestra.run.smithi172.stdout:Modify: 2024-04-22 15:02:53.667254323 +0000 2024-04-22T15:06:15.758 INFO:teuthology.orchestra.run.smithi172.stdout:Change: 2024-04-22 15:02:53.667254323 +0000 2024-04-22T15:06:15.758 INFO:teuthology.orchestra.run.smithi172.stdout: Birth: 2024-04-22 15:02:53.667254323 +0000 2024-04-22T15:06:15.758 DEBUG:teuthology.orchestra.run.smithi172:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-22T15:06:15.856 INFO:teuthology.orchestra.run.smithi172.stderr:1+0 records in 2024-04-22T15:06:15.856 INFO:teuthology.orchestra.run.smithi172.stderr:1+0 records out 2024-04-22T15:06:15.856 INFO:teuthology.orchestra.run.smithi172.stderr:512 bytes copied, 0.000185711 s, 2.8 MB/s 2024-04-22T15:06:15.858 DEBUG:teuthology.orchestra.run.smithi172:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-22T15:06:15.873 DEBUG:teuthology.orchestra.run.smithi172:> stat /dev/vg_nvme/lv_4 2024-04-22T15:06:15.928 INFO:teuthology.orchestra.run.smithi172.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-22T15:06:15.928 INFO:teuthology.orchestra.run.smithi172.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T15:06:15.928 INFO:teuthology.orchestra.run.smithi172.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-04-22T15:06:15.929 INFO:teuthology.orchestra.run.smithi172.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T15:06:15.929 INFO:teuthology.orchestra.run.smithi172.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T15:06:15.929 INFO:teuthology.orchestra.run.smithi172.stdout:Access: 2024-04-22 15:06:15.813698044 +0000 2024-04-22T15:06:15.929 INFO:teuthology.orchestra.run.smithi172.stdout:Modify: 2024-04-22 15:02:54.069244047 +0000 2024-04-22T15:06:15.929 INFO:teuthology.orchestra.run.smithi172.stdout:Change: 2024-04-22 15:02:54.069244047 +0000 2024-04-22T15:06:15.929 INFO:teuthology.orchestra.run.smithi172.stdout: Birth: 2024-04-22 15:02:54.069244047 +0000 2024-04-22T15:06:15.929 DEBUG:teuthology.orchestra.run.smithi172:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-22T15:06:15.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:06:15 smithi172 systemd[1]: Starting Ceph mgr.x for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:06:15.991 INFO:teuthology.orchestra.run.smithi172.stderr:1+0 records in 2024-04-22T15:06:15.992 INFO:teuthology.orchestra.run.smithi172.stderr:1+0 records out 2024-04-22T15:06:15.992 INFO:teuthology.orchestra.run.smithi172.stderr:512 bytes copied, 0.000240981 s, 2.1 MB/s 2024-04-22T15:06:15.993 DEBUG:teuthology.orchestra.run.smithi172:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-22T15:06:16.049 INFO:tasks.cephadm:Deploying osd.0 on smithi145 with /dev/vg_nvme/lv_4... 2024-04-22T15:06:16.049 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-22T15:06:16.353 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:06:16 smithi172 podman[23895]: 2024-04-22 15:06:16.007402736 +0000 UTC m=+0.019130213 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-22T15:06:16.354 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:06:16 smithi172 podman[23895]: 2024-04-22 15:06:16.114023777 +0000 UTC m=+0.125751245 container create 9701c52446e9331b9aee526b80b064e5d71e43d671024fdcbb6e3f0362166caa (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, version=8, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8) 2024-04-22T15:06:16.354 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:06:16 smithi172 podman[23895]: 2024-04-22 15:06:16.350002186 +0000 UTC m=+0.361729661 container init 9701c52446e9331b9aee526b80b064e5d71e43d671024fdcbb6e3f0362166caa (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x, RELEASE=HEAD, release=754, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, version=8, vendor=Red Hat, Inc., io.buildah.version=1.19.8, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-22T15:06:16.652 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:06:16 smithi172 podman[23895]: 2024-04-22 15:06:16.353090162 +0000 UTC m=+0.364817631 container start 9701c52446e9331b9aee526b80b064e5d71e43d671024fdcbb6e3f0362166caa (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , vcs-type=git, RELEASE=HEAD, com.redhat.component=centos-stream-container, distribution-scope=public, version=8, vendor=Red Hat, Inc., GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, io.buildah.version=1.19.8) 2024-04-22T15:06:16.652 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:06:16 smithi172 ceph-mgr[23935]: -- 172.21.15.172:0/562872493 <== mon.2 v2:172.21.15.172:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x562466064340 con 0x56246603d000 2024-04-22T15:06:16.652 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:06:16 smithi172 bash[23895]: 9701c52446e9331b9aee526b80b064e5d71e43d671024fdcbb6e3f0362166caa 2024-04-22T15:06:16.652 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:06:16 smithi172 systemd[1]: Started Ceph mgr.x for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:06:16.652 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:06:16 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:06:16.511+0000 7f818e87c000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T15:06:16.652 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:06:16 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:06:16.648+0000 7f818e87c000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T15:06:16.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:06:16 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:06:16.756+0000 7f818e87c000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T15:06:17.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:17 smithi145 ceph-mon[21899]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:17.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:17 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:17.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:17 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:06:17.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:17 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:17.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:17 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:06:17.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:17 smithi145 ceph-mon[26606]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:17.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:17 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:17.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:17 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:06:17.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:17 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:17.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:17 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:06:17.432 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:06:17 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:06:17.081+0000 7f818e87c000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T15:06:17.433 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:06:17 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:06:17.167+0000 7f818e87c000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T15:06:17.433 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:17 smithi172 ceph-mon[22747]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:17.433 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:17 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:17.433 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:17 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:06:17.433 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:17 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:17.433 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:17 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:06:18.241 INFO:teuthology.orchestra.run.smithi145.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-22T15:06:18.241 INFO:teuthology.orchestra.run.smithi145.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-22T15:06:18.241 INFO:teuthology.orchestra.run.smithi145.stderr: stderr: 10+0 records in 2024-04-22T15:06:18.241 INFO:teuthology.orchestra.run.smithi145.stderr:10+0 records out 2024-04-22T15:06:18.241 INFO:teuthology.orchestra.run.smithi145.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0112616 s, 931 MB/s 2024-04-22T15:06:18.241 INFO:teuthology.orchestra.run.smithi145.stderr:--> Zapping successful for: 2024-04-22T15:06:18.693 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:18 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:18.693 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:18 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:18.693 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:18 smithi145 ceph-mon[21899]: Reconfiguring mgr.y (unknown last config time)... 2024-04-22T15:06:18.693 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:18 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:06:18.693 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:18 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:06:18.693 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:18 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:18.693 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:18 smithi145 ceph-mon[21899]: Reconfiguring daemon mgr.y on smithi145 2024-04-22T15:06:18.693 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:18 smithi145 ceph-mon[21899]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:18.694 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:18 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:18.694 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:18 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:18.694 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:18 smithi145 ceph-mon[26606]: Reconfiguring mgr.y (unknown last config time)... 2024-04-22T15:06:18.694 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:18 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:06:18.694 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:18 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:06:18.694 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:18 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:18.694 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:18 smithi145 ceph-mon[26606]: Reconfiguring daemon mgr.y on smithi145 2024-04-22T15:06:18.694 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:18 smithi145 ceph-mon[26606]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:18.718 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:06:18 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:06:18.318+0000 7f818e87c000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T15:06:18.718 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:06:18 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:06:18.482+0000 7f818e87c000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T15:06:18.718 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:06:18 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:06:18.563+0000 7f818e87c000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T15:06:18.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:18 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:18.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:18 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:18.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:18 smithi172 ceph-mon[22747]: Reconfiguring mgr.y (unknown last config time)... 2024-04-22T15:06:18.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:18 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:06:18.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:18 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:06:18.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:18 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:18.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:18 smithi172 ceph-mon[22747]: Reconfiguring daemon mgr.y on smithi145 2024-04-22T15:06:18.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:18 smithi172 ceph-mon[22747]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:19.218 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:06:18 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:06:18.872+0000 7f818e87c000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T15:06:19.218 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:06:18 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:06:18.962+0000 7f818e87c000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T15:06:19.218 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:06:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:06:19.053+0000 7f818e87c000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T15:06:19.352 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi145:vg_nvme/lv_4 2024-04-22T15:06:19.718 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:06:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:06:19.292+0000 7f818e87c000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T15:06:20.211 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:06:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:06:19.724+0000 7f818e87c000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T15:06:20.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:06:20 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:06:20.207+0000 7f818e87c000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T15:06:20.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:06:20 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:06:20.299+0000 7f818e87c000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T15:06:20.918 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:20 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:20.919 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:20 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:06:20.919 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:20 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:20.919 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:20 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:06:20.919 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:20 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:20.919 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:20 smithi145 ceph-mon[26606]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:20.919 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:20 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:06:20.919 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:20 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:06:20.919 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:20 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:20.919 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:20 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:20.920 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:20 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:06:20.920 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:20 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:20.920 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:20 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:06:20.920 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:20 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:20.920 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:20 smithi145 ceph-mon[21899]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:20.920 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:20 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:06:20.920 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:20 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:06:20.920 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:20 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:20.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:06:20 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:06:20.587+0000 7f818e87c000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T15:06:20.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:06:20 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:06:20.675+0000 7f818e87c000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T15:06:20.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:20 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:20.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:20 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:06:20.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:20 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:20.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:20 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:06:20.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:20 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:20.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:20 smithi172 ceph-mon[22747]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:20.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:20 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:06:20.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:20 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:06:20.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:20 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:21.416 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:06:21 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:06:21.080+0000 7f818e87c000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T15:06:21.416 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:06:21 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:06:21.210+0000 7f818e87c000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T15:06:21.416 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:06:21 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:06:21.301+0000 7f818e87c000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T15:06:21.416 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:06:21 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:06:21.412+0000 7f818e87c000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T15:06:21.718 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:06:21 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:06:21.500+0000 7f818e87c000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T15:06:21.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:21 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:06:21.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:21 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:06:21.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:21 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:21.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:21 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:06:21.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:21 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:06:21.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:21 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:22.218 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:06:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:06:22.042+0000 7f818e87c000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T15:06:22.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:21 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:06:22.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:21 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:06:22.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:21 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:22.718 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:06:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:06:22.402+0000 7f818e87c000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T15:06:22.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:22 smithi145 ceph-mon[21899]: from='client.24124 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:06:22.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:22 smithi145 ceph-mon[21899]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:22.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:22 smithi145 ceph-mon[21899]: Standby manager daemon x started 2024-04-22T15:06:22.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:22 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.172:0/3446963370' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T15:06:22.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:22 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.172:0/3446963370' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:06:22.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:22 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.172:0/3446963370' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T15:06:22.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:22 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.172:0/3446963370' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:06:22.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:22 smithi145 ceph-mon[26606]: from='client.24124 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:06:22.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:22 smithi145 ceph-mon[26606]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:22.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:22 smithi145 ceph-mon[26606]: Standby manager daemon x started 2024-04-22T15:06:22.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:22 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.172:0/3446963370' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T15:06:22.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:22 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.172:0/3446963370' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:06:22.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:22 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.172:0/3446963370' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T15:06:22.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:22 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.172:0/3446963370' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:06:23.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:22 smithi172 ceph-mon[22747]: from='client.24124 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:06:23.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:22 smithi172 ceph-mon[22747]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:23.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:22 smithi172 ceph-mon[22747]: Standby manager daemon x started 2024-04-22T15:06:23.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:22 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.172:0/3446963370' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T15:06:23.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:22 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.172:0/3446963370' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:06:23.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:22 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.172:0/3446963370' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T15:06:23.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:22 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.172:0/3446963370' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:06:24.111 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:23 smithi145 ceph-mon[21899]: mgrmap e14: y(active, since 63s), standbys: x 2024-04-22T15:06:24.111 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:23 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T15:06:24.112 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:23 smithi145 ceph-mon[26606]: mgrmap e14: y(active, since 63s), standbys: x 2024-04-22T15:06:24.112 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:23 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T15:06:24.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:23 smithi172 ceph-mon[22747]: mgrmap e14: y(active, since 63s), standbys: x 2024-04-22T15:06:24.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:23 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T15:06:25.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:24 smithi172 ceph-mon[22747]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:25.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:24 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2840179882' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f61de15-c199-4966-8d5d-8acd8ca2d6c8"}]: dispatch 2024-04-22T15:06:25.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:24 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2840179882' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8f61de15-c199-4966-8d5d-8acd8ca2d6c8"}]': finished 2024-04-22T15:06:25.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:24 smithi172 ceph-mon[22747]: osdmap e5: 1 total, 0 up, 1 in 2024-04-22T15:06:25.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:24 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:06:25.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:24 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/3075688181' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:06:25.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:24 smithi145 ceph-mon[21899]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:25.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:24 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2840179882' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f61de15-c199-4966-8d5d-8acd8ca2d6c8"}]: dispatch 2024-04-22T15:06:25.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:24 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2840179882' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8f61de15-c199-4966-8d5d-8acd8ca2d6c8"}]': finished 2024-04-22T15:06:25.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:24 smithi145 ceph-mon[21899]: osdmap e5: 1 total, 0 up, 1 in 2024-04-22T15:06:25.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:24 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:06:25.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:24 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3075688181' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:06:25.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:24 smithi145 ceph-mon[26606]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:25.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:24 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2840179882' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f61de15-c199-4966-8d5d-8acd8ca2d6c8"}]: dispatch 2024-04-22T15:06:25.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:24 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2840179882' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8f61de15-c199-4966-8d5d-8acd8ca2d6c8"}]': finished 2024-04-22T15:06:25.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:24 smithi145 ceph-mon[26606]: osdmap e5: 1 total, 0 up, 1 in 2024-04-22T15:06:25.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:24 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:06:25.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:24 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3075688181' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:06:27.077 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:26 smithi145 ceph-mon[21899]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:27.078 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:26 smithi145 ceph-mon[26606]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:27.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:26 smithi172 ceph-mon[22747]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:28.984 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:28 smithi145 ceph-mon[21899]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:28.985 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:28 smithi145 ceph-mon[26606]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:29.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:28 smithi172 ceph-mon[22747]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:31.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:30 smithi145 ceph-mon[21899]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:31.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:30 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T15:06:31.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:30 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:31.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:30 smithi145 ceph-mon[21899]: Deploying daemon osd.0 on smithi145 2024-04-22T15:06:31.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:30 smithi145 ceph-mon[26606]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:31.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:30 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T15:06:31.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:30 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:31.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:30 smithi145 ceph-mon[26606]: Deploying daemon osd.0 on smithi145 2024-04-22T15:06:31.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:30 smithi172 ceph-mon[22747]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:31.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:30 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T15:06:31.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:30 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:31.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:30 smithi172 ceph-mon[22747]: Deploying daemon osd.0 on smithi145 2024-04-22T15:06:33.085 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:32 smithi145 ceph-mon[21899]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:33.086 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:32 smithi145 ceph-mon[26606]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:33.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:32 smithi172 ceph-mon[22747]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:35.168 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:34 smithi145 ceph-mon[21899]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:35.168 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:34 smithi145 ceph-mon[26606]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:35.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:34 smithi172 ceph-mon[22747]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:36.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:36 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:36.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:36 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:06:36.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:36 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:36.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:36 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:06:36.503 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:36 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:36.503 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:36 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:06:36.504 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:36 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:36.504 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:36 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:06:36.504 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:36 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:36.504 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:36 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:06:36.504 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:36 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:36.504 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:36 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:06:37.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:37 smithi172 ceph-mon[22747]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:37.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:37 smithi145 ceph-mon[21899]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:37 smithi145 ceph-mon[26606]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:38.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:38 smithi172 ceph-mon[22747]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:38.504 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:38 smithi145 ceph-mon[21899]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:38.505 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:38 smithi145 ceph-mon[26606]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:38.534 INFO:teuthology.orchestra.run.smithi145.stdout:Created osd(s) 0 on host 'smithi145' 2024-04-22T15:06:39.247 DEBUG:teuthology.orchestra.run.smithi145:osd.0> sudo journalctl -f -n 0 -u ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.0.service 2024-04-22T15:06:39.249 INFO:tasks.cephadm:Deploying osd.1 on smithi145 with /dev/vg_nvme/lv_3... 2024-04-22T15:06:39.249 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-22T15:06:39.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:39 smithi172 ceph-mon[22747]: from='osd.0 [v2:172.21.15.145:6802/2466746205,v1:172.21.15.145:6803/2466746205]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-22T15:06:39.470 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:39 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:39.470 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:39 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:39.600 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:39 smithi145 ceph-mon[26606]: from='osd.0 [v2:172.21.15.145:6802/2466746205,v1:172.21.15.145:6803/2466746205]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-22T15:06:39.600 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:39 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:39.601 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:39 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:39.601 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:39 smithi145 ceph-mon[21899]: from='osd.0 [v2:172.21.15.145:6802/2466746205,v1:172.21.15.145:6803/2466746205]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-22T15:06:39.601 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:39 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:39.601 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:39 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:40.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:40 smithi172 ceph-mon[22747]: from='osd.0 [v2:172.21.15.145:6802/2466746205,v1:172.21.15.145:6803/2466746205]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-22T15:06:40.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:40 smithi172 ceph-mon[22747]: osdmap e6: 1 total, 0 up, 1 in 2024-04-22T15:06:40.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:40 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:06:40.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:40 smithi172 ceph-mon[22747]: from='osd.0 [v2:172.21.15.145:6802/2466746205,v1:172.21.15.145:6803/2466746205]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:06:40.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:40 smithi172 ceph-mon[22747]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:40.523 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:40 smithi145 ceph-mon[26606]: from='osd.0 [v2:172.21.15.145:6802/2466746205,v1:172.21.15.145:6803/2466746205]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-22T15:06:40.523 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:40 smithi145 ceph-mon[26606]: osdmap e6: 1 total, 0 up, 1 in 2024-04-22T15:06:40.523 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:40 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:06:40.523 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:40 smithi145 ceph-mon[26606]: from='osd.0 [v2:172.21.15.145:6802/2466746205,v1:172.21.15.145:6803/2466746205]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:06:40.523 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:40 smithi145 ceph-mon[26606]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:40.524 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:40 smithi145 ceph-mon[21899]: from='osd.0 [v2:172.21.15.145:6802/2466746205,v1:172.21.15.145:6803/2466746205]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-22T15:06:40.524 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:40 smithi145 ceph-mon[21899]: osdmap e6: 1 total, 0 up, 1 in 2024-04-22T15:06:40.524 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:40 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:06:40.524 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:40 smithi145 ceph-mon[21899]: from='osd.0 [v2:172.21.15.145:6802/2466746205,v1:172.21.15.145:6803/2466746205]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:06:40.524 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:40 smithi145 ceph-mon[21899]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:40.524 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:06:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0[29461]: 2024-04-22T15:06:40.204+0000 7f0d8a35b700 -1 osd.0 0 waiting for initial osdmap 2024-04-22T15:06:40.524 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:06:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0[29461]: 2024-04-22T15:06:40.206+0000 7f0d84cf2700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T15:06:41.226 INFO:teuthology.orchestra.run.smithi145.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-22T15:06:41.226 INFO:teuthology.orchestra.run.smithi145.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-22T15:06:41.226 INFO:teuthology.orchestra.run.smithi145.stderr: stderr: 10+0 records in 2024-04-22T15:06:41.226 INFO:teuthology.orchestra.run.smithi145.stderr:10+0 records out 2024-04-22T15:06:41.226 INFO:teuthology.orchestra.run.smithi145.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0129002 s, 813 MB/s 2024-04-22T15:06:41.226 INFO:teuthology.orchestra.run.smithi145.stderr:--> Zapping successful for: 2024-04-22T15:06:41.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:41 smithi172 ceph-mon[22747]: from='osd.0 [v2:172.21.15.145:6802/2466746205,v1:172.21.15.145:6803/2466746205]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-04-22T15:06:41.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:41 smithi172 ceph-mon[22747]: osdmap e7: 1 total, 0 up, 1 in 2024-04-22T15:06:41.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:41 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:06:41.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:41 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:06:41.489 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:41 smithi145 ceph-mon[21899]: from='osd.0 [v2:172.21.15.145:6802/2466746205,v1:172.21.15.145:6803/2466746205]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-04-22T15:06:41.489 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:41 smithi145 ceph-mon[21899]: osdmap e7: 1 total, 0 up, 1 in 2024-04-22T15:06:41.489 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:41 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:06:41.489 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:41 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:06:41.489 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:41 smithi145 ceph-mon[26606]: from='osd.0 [v2:172.21.15.145:6802/2466746205,v1:172.21.15.145:6803/2466746205]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-04-22T15:06:41.489 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:41 smithi145 ceph-mon[26606]: osdmap e7: 1 total, 0 up, 1 in 2024-04-22T15:06:41.489 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:41 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:06:41.490 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:41 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:06:41.871 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi145:vg_nvme/lv_3 2024-04-22T15:06:42.405 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:42 smithi145 ceph-mon[21899]: purged_snaps scrub starts 2024-04-22T15:06:42.405 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:42 smithi145 ceph-mon[21899]: purged_snaps scrub ok 2024-04-22T15:06:42.405 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:42 smithi145 ceph-mon[21899]: osd.0 [v2:172.21.15.145:6802/2466746205,v1:172.21.15.145:6803/2466746205] boot 2024-04-22T15:06:42.405 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:42 smithi145 ceph-mon[21899]: osdmap e8: 1 total, 1 up, 1 in 2024-04-22T15:06:42.405 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:42 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:06:42.406 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:42 smithi145 ceph-mon[21899]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:42.406 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:42 smithi145 ceph-mon[26606]: purged_snaps scrub starts 2024-04-22T15:06:42.406 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:42 smithi145 ceph-mon[26606]: purged_snaps scrub ok 2024-04-22T15:06:42.406 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:42 smithi145 ceph-mon[26606]: osd.0 [v2:172.21.15.145:6802/2466746205,v1:172.21.15.145:6803/2466746205] boot 2024-04-22T15:06:42.406 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:42 smithi145 ceph-mon[26606]: osdmap e8: 1 total, 1 up, 1 in 2024-04-22T15:06:42.406 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:42 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:06:42.406 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:42 smithi145 ceph-mon[26606]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:42.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:42 smithi172 ceph-mon[22747]: purged_snaps scrub starts 2024-04-22T15:06:42.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:42 smithi172 ceph-mon[22747]: purged_snaps scrub ok 2024-04-22T15:06:42.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:42 smithi172 ceph-mon[22747]: osd.0 [v2:172.21.15.145:6802/2466746205,v1:172.21.15.145:6803/2466746205] boot 2024-04-22T15:06:42.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:42 smithi172 ceph-mon[22747]: osdmap e8: 1 total, 1 up, 1 in 2024-04-22T15:06:42.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:42 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:06:42.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:42 smithi172 ceph-mon[22747]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T15:06:43.350 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:43 smithi145 ceph-mon[21899]: osdmap e9: 1 total, 1 up, 1 in 2024-04-22T15:06:43.351 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:43 smithi145 ceph-mon[26606]: osdmap e9: 1 total, 1 up, 1 in 2024-04-22T15:06:43.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:43 smithi172 ceph-mon[22747]: osdmap e9: 1 total, 1 up, 1 in 2024-04-22T15:06:45.165 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:44 smithi145 ceph-mon[21899]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:06:45.165 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:44 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:45.166 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:44 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:06:45.166 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:44 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:45.166 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:44 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:45.166 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:44 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:06:45.166 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:44 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:45.166 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:44 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:06:45.166 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:44 smithi145 ceph-mon[26606]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:06:45.166 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:44 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:45.166 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:44 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:06:45.166 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:44 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:45.166 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:44 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:45.167 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:44 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:06:45.167 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:44 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:45.167 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:44 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:06:45.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:44 smithi172 ceph-mon[22747]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:06:45.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:44 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:45.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:44 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:06:45.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:44 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:45.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:44 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:45.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:44 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:06:45.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:44 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:45.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:44 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:06:46.095 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:45 smithi145 ceph-mon[26606]: Detected new or changed devices on smithi145 2024-04-22T15:06:46.095 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:45 smithi145 ceph-mon[26606]: Adjusting osd_memory_target on smithi145 to 16064M 2024-04-22T15:06:46.095 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:45 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:46.095 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:45 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:06:46.095 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:45 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:06:46.095 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:45 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:46.096 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:45 smithi145 ceph-mon[21899]: Detected new or changed devices on smithi145 2024-04-22T15:06:46.096 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:45 smithi145 ceph-mon[21899]: Adjusting osd_memory_target on smithi145 to 16064M 2024-04-22T15:06:46.096 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:45 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:46.096 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:45 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:06:46.096 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:45 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:06:46.096 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:45 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:46.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:45 smithi172 ceph-mon[22747]: Detected new or changed devices on smithi145 2024-04-22T15:06:46.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:45 smithi172 ceph-mon[22747]: Adjusting osd_memory_target on smithi145 to 16064M 2024-04-22T15:06:46.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:45 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:06:46.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:45 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:06:46.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:45 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:06:46.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:45 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:46.924 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:46 smithi145 ceph-mon[21899]: from='client.14247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:06:46.924 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:46 smithi145 ceph-mon[21899]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:06:46.925 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:46 smithi145 ceph-mon[26606]: from='client.14247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:06:46.925 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:46 smithi145 ceph-mon[26606]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:06:47.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:46 smithi172 ceph-mon[22747]: from='client.14247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:06:47.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:46 smithi172 ceph-mon[22747]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:06:48.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:47 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/3428109889' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f863b9ae-7cb4-4b74-a2a1-a2384e809318"}]: dispatch 2024-04-22T15:06:48.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:47 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/3428109889' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f863b9ae-7cb4-4b74-a2a1-a2384e809318"}]': finished 2024-04-22T15:06:48.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:47 smithi172 ceph-mon[22747]: osdmap e10: 2 total, 1 up, 2 in 2024-04-22T15:06:48.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:47 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:06:48.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:47 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3428109889' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f863b9ae-7cb4-4b74-a2a1-a2384e809318"}]: dispatch 2024-04-22T15:06:48.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:47 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3428109889' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f863b9ae-7cb4-4b74-a2a1-a2384e809318"}]': finished 2024-04-22T15:06:48.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:47 smithi145 ceph-mon[26606]: osdmap e10: 2 total, 1 up, 2 in 2024-04-22T15:06:48.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:47 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:06:48.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:47 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3428109889' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f863b9ae-7cb4-4b74-a2a1-a2384e809318"}]: dispatch 2024-04-22T15:06:48.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:47 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3428109889' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f863b9ae-7cb4-4b74-a2a1-a2384e809318"}]': finished 2024-04-22T15:06:48.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:47 smithi145 ceph-mon[21899]: osdmap e10: 2 total, 1 up, 2 in 2024-04-22T15:06:48.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:47 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:06:49.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:48 smithi172 ceph-mon[22747]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:06:49.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:48 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/3309842637' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:06:49.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:48 smithi145 ceph-mon[21899]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:06:49.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:48 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3309842637' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:06:49.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:48 smithi145 ceph-mon[26606]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:06:49.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:48 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3309842637' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:06:51.091 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:50 smithi145 ceph-mon[21899]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:06:51.091 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:50 smithi145 ceph-mon[26606]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:06:51.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:50 smithi172 ceph-mon[22747]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:06:53.071 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:52 smithi145 ceph-mon[21899]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:06:53.072 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:52 smithi145 ceph-mon[26606]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:06:53.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:52 smithi172 ceph-mon[22747]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:06:55.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:54 smithi145 ceph-mon[21899]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:06:55.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:54 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T15:06:55.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:54 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:55.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:54 smithi145 ceph-mon[26606]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:06:55.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:54 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T15:06:55.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:54 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:55.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:54 smithi172 ceph-mon[22747]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:06:55.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:54 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T15:06:55.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:54 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:06:56.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:55 smithi145 ceph-mon[21899]: Deploying daemon osd.1 on smithi145 2024-04-22T15:06:56.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:55 smithi145 ceph-mon[26606]: Deploying daemon osd.1 on smithi145 2024-04-22T15:06:56.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:55 smithi172 ceph-mon[22747]: Deploying daemon osd.1 on smithi145 2024-04-22T15:06:57.151 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:56 smithi145 ceph-mon[21899]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:06:57.151 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:56 smithi145 ceph-mon[26606]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:06:57.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:56 smithi172 ceph-mon[22747]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:06:59.169 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:06:58 smithi145 ceph-mon[21899]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:06:59.169 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:06:58 smithi145 ceph-mon[26606]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:06:59.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:06:58 smithi172 ceph-mon[22747]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:00.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:00 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:00.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:00 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:07:00.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:00 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:00.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:00 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:07:00.554 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:00 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:00.554 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:00 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:07:00.554 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:00 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:00.554 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:00 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:07:00.554 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:00 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:00.554 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:00 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:07:00.554 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:00 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:00.554 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:00 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:07:01.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:01 smithi172 ceph-mon[22747]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:01.469 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:01 smithi145 ceph-mon[21899]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:01.469 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:01 smithi145 ceph-mon[26606]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:02.419 INFO:teuthology.orchestra.run.smithi145.stdout:Created osd(s) 1 on host 'smithi145' 2024-04-22T15:07:03.034 DEBUG:teuthology.orchestra.run.smithi145:osd.1> sudo journalctl -f -n 0 -u ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.1.service 2024-04-22T15:07:03.036 INFO:tasks.cephadm:Deploying osd.2 on smithi145 with /dev/vg_nvme/lv_2... 2024-04-22T15:07:03.037 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-22T15:07:03.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:03 smithi145 ceph-mon[26606]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:03.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:03 smithi145 ceph-mon[26606]: from='osd.1 [v2:172.21.15.145:6810/1586792147,v1:172.21.15.145:6811/1586792147]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T15:07:03.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:03 smithi145 ceph-mon[26606]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T15:07:03.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:03 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:03.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:03 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:03.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:03 smithi145 ceph-mon[21899]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:03.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:03 smithi145 ceph-mon[21899]: from='osd.1 [v2:172.21.15.145:6810/1586792147,v1:172.21.15.145:6811/1586792147]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T15:07:03.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:03 smithi145 ceph-mon[21899]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T15:07:03.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:03 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:03.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:03 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:03.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:03 smithi172 ceph-mon[22747]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:03.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:03 smithi172 ceph-mon[22747]: from='osd.1 [v2:172.21.15.145:6810/1586792147,v1:172.21.15.145:6811/1586792147]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T15:07:03.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:03 smithi172 ceph-mon[22747]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T15:07:03.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:03 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:03.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:03 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:04.351 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:04 smithi145 ceph-mon[21899]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-22T15:07:04.352 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:04 smithi145 ceph-mon[21899]: osdmap e11: 2 total, 1 up, 2 in 2024-04-22T15:07:04.352 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:04 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:07:04.352 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:04 smithi145 ceph-mon[21899]: from='osd.1 [v2:172.21.15.145:6810/1586792147,v1:172.21.15.145:6811/1586792147]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:07:04.352 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:04 smithi145 ceph-mon[21899]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:07:04.352 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:04 smithi145 ceph-mon[21899]: pgmap v53: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:04.352 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:04 smithi145 ceph-mon[26606]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-22T15:07:04.352 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:04 smithi145 ceph-mon[26606]: osdmap e11: 2 total, 1 up, 2 in 2024-04-22T15:07:04.352 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:04 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:07:04.352 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:04 smithi145 ceph-mon[26606]: from='osd.1 [v2:172.21.15.145:6810/1586792147,v1:172.21.15.145:6811/1586792147]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:07:04.353 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:04 smithi145 ceph-mon[26606]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:07:04.353 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:04 smithi145 ceph-mon[26606]: pgmap v53: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:04.353 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:07:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1[32422]: 2024-04-22T15:07:04.192+0000 7f8b660b0700 -1 osd.1 0 waiting for initial osdmap 2024-04-22T15:07:04.353 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:07:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1[32422]: 2024-04-22T15:07:04.194+0000 7f8b5ea43700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T15:07:04.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:04 smithi172 ceph-mon[22747]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-22T15:07:04.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:04 smithi172 ceph-mon[22747]: osdmap e11: 2 total, 1 up, 2 in 2024-04-22T15:07:04.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:04 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:07:04.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:04 smithi172 ceph-mon[22747]: from='osd.1 [v2:172.21.15.145:6810/1586792147,v1:172.21.15.145:6811/1586792147]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:07:04.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:04 smithi172 ceph-mon[22747]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:07:04.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:04 smithi172 ceph-mon[22747]: pgmap v53: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:05 smithi172 ceph-mon[22747]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-04-22T15:07:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:05 smithi172 ceph-mon[22747]: osdmap e12: 2 total, 1 up, 2 in 2024-04-22T15:07:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:05 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:07:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:05 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:07:05.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:05 smithi145 ceph-mon[21899]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-04-22T15:07:05.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:05 smithi145 ceph-mon[21899]: osdmap e12: 2 total, 1 up, 2 in 2024-04-22T15:07:05.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:05 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:07:05.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:05 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:07:05.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:05 smithi145 ceph-mon[26606]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-04-22T15:07:05.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:05 smithi145 ceph-mon[26606]: osdmap e12: 2 total, 1 up, 2 in 2024-04-22T15:07:05.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:05 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:07:05.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:05 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:07:05.679 INFO:teuthology.orchestra.run.smithi145.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-22T15:07:05.680 INFO:teuthology.orchestra.run.smithi145.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-22T15:07:05.680 INFO:teuthology.orchestra.run.smithi145.stderr: stderr: 10+0 records in 2024-04-22T15:07:05.680 INFO:teuthology.orchestra.run.smithi145.stderr:10+0 records out 2024-04-22T15:07:05.680 INFO:teuthology.orchestra.run.smithi145.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0125181 s, 838 MB/s 2024-04-22T15:07:05.680 INFO:teuthology.orchestra.run.smithi145.stderr:--> Zapping successful for: 2024-04-22T15:07:06.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:06 smithi145 ceph-mon[21899]: purged_snaps scrub starts 2024-04-22T15:07:06.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:06 smithi145 ceph-mon[21899]: purged_snaps scrub ok 2024-04-22T15:07:06.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:06 smithi145 ceph-mon[21899]: osd.1 [v2:172.21.15.145:6810/1586792147,v1:172.21.15.145:6811/1586792147] boot 2024-04-22T15:07:06.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:06 smithi145 ceph-mon[21899]: osdmap e13: 2 total, 2 up, 2 in 2024-04-22T15:07:06.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:06 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:07:06.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:06 smithi145 ceph-mon[21899]: pgmap v56: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:06.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:06 smithi145 ceph-mon[26606]: purged_snaps scrub starts 2024-04-22T15:07:06.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:06 smithi145 ceph-mon[26606]: purged_snaps scrub ok 2024-04-22T15:07:06.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:06 smithi145 ceph-mon[26606]: osd.1 [v2:172.21.15.145:6810/1586792147,v1:172.21.15.145:6811/1586792147] boot 2024-04-22T15:07:06.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:06 smithi145 ceph-mon[26606]: osdmap e13: 2 total, 2 up, 2 in 2024-04-22T15:07:06.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:06 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:07:06.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:06 smithi145 ceph-mon[26606]: pgmap v56: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:06.412 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi145:vg_nvme/lv_2 2024-04-22T15:07:06.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:06 smithi172 ceph-mon[22747]: purged_snaps scrub starts 2024-04-22T15:07:06.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:06 smithi172 ceph-mon[22747]: purged_snaps scrub ok 2024-04-22T15:07:06.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:06 smithi172 ceph-mon[22747]: osd.1 [v2:172.21.15.145:6810/1586792147,v1:172.21.15.145:6811/1586792147] boot 2024-04-22T15:07:06.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:06 smithi172 ceph-mon[22747]: osdmap e13: 2 total, 2 up, 2 in 2024-04-22T15:07:06.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:06 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:07:06.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:06 smithi172 ceph-mon[22747]: pgmap v56: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T15:07:07.304 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:07 smithi145 ceph-mon[21899]: osdmap e14: 2 total, 2 up, 2 in 2024-04-22T15:07:07.304 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:07 smithi145 ceph-mon[26606]: osdmap e14: 2 total, 2 up, 2 in 2024-04-22T15:07:07.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:07 smithi172 ceph-mon[22747]: osdmap e14: 2 total, 2 up, 2 in 2024-04-22T15:07:08.436 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:08 smithi145 ceph-mon[21899]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:08.436 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:08 smithi145 ceph-mon[26606]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:08.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:08 smithi172 ceph-mon[22747]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:09.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:09 smithi172 ceph-mon[22747]: from='client.24178 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:07:09.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:09 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:07:09.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:09 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:07:09.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:09 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:09.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:09 smithi172 ceph-mon[22747]: Detected new or changed devices on smithi145 2024-04-22T15:07:09.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:09 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:09.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:09 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:09.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:09 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:09.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:09 smithi172 ceph-mon[22747]: Adjusting osd_memory_target on smithi145 to 8032M 2024-04-22T15:07:09.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:09 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:09.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:09 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:09.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:09 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:07:09.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:09 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:09.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:09 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:07:09.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:09 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:10.164 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:09 smithi145 ceph-mon[21899]: from='client.24178 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:07:10.164 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:09 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:07:10.164 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:09 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:07:10.164 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:09 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:10.164 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:09 smithi145 ceph-mon[21899]: Detected new or changed devices on smithi145 2024-04-22T15:07:10.164 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:09 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:10.164 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:09 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:10.165 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:09 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:10.165 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:09 smithi145 ceph-mon[21899]: Adjusting osd_memory_target on smithi145 to 8032M 2024-04-22T15:07:10.165 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:09 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:10.165 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:09 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:10.165 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:09 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:07:10.165 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:09 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:10.165 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:09 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:07:10.165 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:09 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:10.165 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:09 smithi145 ceph-mon[26606]: from='client.24178 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:07:10.165 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:09 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:07:10.165 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:09 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:07:10.166 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:09 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:10.166 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:09 smithi145 ceph-mon[26606]: Detected new or changed devices on smithi145 2024-04-22T15:07:10.166 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:09 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:10.166 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:09 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:10.166 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:09 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:10.166 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:09 smithi145 ceph-mon[26606]: Adjusting osd_memory_target on smithi145 to 8032M 2024-04-22T15:07:10.166 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:09 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:10.166 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:09 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:10.166 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:09 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:07:10.166 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:09 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:10.166 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:09 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:07:10.166 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:09 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:11.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:11 smithi172 ceph-mon[22747]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:11.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:11 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/3372878504' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "359c8817-27db-4026-891b-909210f255bb"}]: dispatch 2024-04-22T15:07:11.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:11 smithi172 ceph-mon[22747]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "359c8817-27db-4026-891b-909210f255bb"}]: dispatch 2024-04-22T15:07:11.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:11 smithi172 ceph-mon[22747]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "359c8817-27db-4026-891b-909210f255bb"}]': finished 2024-04-22T15:07:11.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:11 smithi172 ceph-mon[22747]: osdmap e15: 3 total, 2 up, 3 in 2024-04-22T15:07:11.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:11 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:07:11.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:11 smithi145 ceph-mon[21899]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:11.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:11 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3372878504' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "359c8817-27db-4026-891b-909210f255bb"}]: dispatch 2024-04-22T15:07:11.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:11 smithi145 ceph-mon[21899]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "359c8817-27db-4026-891b-909210f255bb"}]: dispatch 2024-04-22T15:07:11.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:11 smithi145 ceph-mon[21899]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "359c8817-27db-4026-891b-909210f255bb"}]': finished 2024-04-22T15:07:11.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:11 smithi145 ceph-mon[21899]: osdmap e15: 3 total, 2 up, 3 in 2024-04-22T15:07:11.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:11 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:07:11.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:11 smithi145 ceph-mon[26606]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:11.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:11 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3372878504' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "359c8817-27db-4026-891b-909210f255bb"}]: dispatch 2024-04-22T15:07:11.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:11 smithi145 ceph-mon[26606]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "359c8817-27db-4026-891b-909210f255bb"}]: dispatch 2024-04-22T15:07:11.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:11 smithi145 ceph-mon[26606]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "359c8817-27db-4026-891b-909210f255bb"}]': finished 2024-04-22T15:07:11.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:11 smithi145 ceph-mon[26606]: osdmap e15: 3 total, 2 up, 3 in 2024-04-22T15:07:11.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:11 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:07:12.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:12 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/987185996' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:07:12.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:12 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/987185996' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:07:12.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:12 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/987185996' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:07:13.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:13 smithi145 ceph-mon[21899]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:13.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:13 smithi145 ceph-mon[26606]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:13.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:13 smithi172 ceph-mon[22747]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:15.038 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:14 smithi145 ceph-mon[21899]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:15.038 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:14 smithi145 ceph-mon[26606]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:15.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:14 smithi172 ceph-mon[22747]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:17.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:16 smithi145 ceph-mon[21899]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:17.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:16 smithi145 ceph-mon[26606]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:17.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:16 smithi172 ceph-mon[22747]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:18.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:17 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T15:07:18.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:17 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:18.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:17 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T15:07:18.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:17 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:18.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:17 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T15:07:18.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:17 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:19.096 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:18 smithi145 ceph-mon[21899]: Deploying daemon osd.2 on smithi145 2024-04-22T15:07:19.096 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:18 smithi145 ceph-mon[21899]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:19.096 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:18 smithi145 ceph-mon[26606]: Deploying daemon osd.2 on smithi145 2024-04-22T15:07:19.097 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:18 smithi145 ceph-mon[26606]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:19.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:18 smithi172 ceph-mon[22747]: Deploying daemon osd.2 on smithi145 2024-04-22T15:07:19.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:18 smithi172 ceph-mon[22747]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:20.987 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:20 smithi145 ceph-mon[26606]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:20.988 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:20 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:07:20.988 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:20 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:07:20.988 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:20 smithi145 ceph-mon[21899]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:20.988 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:20 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:07:20.988 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:20 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:07:21.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:20 smithi172 ceph-mon[22747]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:21.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:20 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:07:21.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:20 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:07:23.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:22 smithi172 ceph-mon[22747]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:23.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:22 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:23.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:22 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:07:23.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:22 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:23.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:22 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:07:23.240 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:22 smithi145 ceph-mon[21899]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:23.240 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:22 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:23.240 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:22 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:07:23.240 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:22 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:23.241 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:22 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:07:23.241 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:22 smithi145 ceph-mon[26606]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:23.241 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:22 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:23.241 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:22 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:07:23.241 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:22 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:23.241 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:22 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:07:25.185 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:24 smithi145 ceph-mon[21899]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:25.186 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:24 smithi145 ceph-mon[26606]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:25.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:24 smithi172 ceph-mon[22747]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:25.976 INFO:teuthology.orchestra.run.smithi145.stdout:Created osd(s) 2 on host 'smithi145' 2024-04-22T15:07:26.196 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:26 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:26.196 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:26 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:26.196 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:26 smithi145 ceph-mon[26606]: from='osd.2 [v2:172.21.15.145:6818/3315449646,v1:172.21.15.145:6819/3315449646]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-22T15:07:26.197 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:26 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:26.197 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:26 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:26.197 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:26 smithi145 ceph-mon[21899]: from='osd.2 [v2:172.21.15.145:6818/3315449646,v1:172.21.15.145:6819/3315449646]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-22T15:07:26.197 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:26 smithi145 ceph-mon[21899]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-22T15:07:26.197 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:26 smithi145 ceph-mon[21899]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:26.197 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:26 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:26.197 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:26 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:07:26.197 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:26 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:26.197 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:26 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:07:26.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:26 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:26.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:26 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:26.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:26 smithi172 ceph-mon[22747]: from='osd.2 [v2:172.21.15.145:6818/3315449646,v1:172.21.15.145:6819/3315449646]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-22T15:07:26.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:26 smithi172 ceph-mon[22747]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-22T15:07:26.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:26 smithi172 ceph-mon[22747]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:26.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:26 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:26.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:26 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:07:26.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:26 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:26.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:26 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:07:26.616 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:26 smithi145 ceph-mon[26606]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-22T15:07:26.616 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:26 smithi145 ceph-mon[26606]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:26.616 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:26 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:26.616 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:26 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:07:26.616 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:26 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:26.617 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:26 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:07:26.639 DEBUG:teuthology.orchestra.run.smithi145:osd.2> sudo journalctl -f -n 0 -u ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.2.service 2024-04-22T15:07:26.641 INFO:tasks.cephadm:Deploying osd.3 on smithi145 with /dev/vg_nvme/lv_1... 2024-04-22T15:07:26.642 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-22T15:07:27.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:27 smithi172 ceph-mon[22747]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-22T15:07:27.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:27 smithi172 ceph-mon[22747]: osdmap e16: 3 total, 2 up, 3 in 2024-04-22T15:07:27.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:27 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:07:27.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:27 smithi172 ceph-mon[22747]: from='osd.2 [v2:172.21.15.145:6818/3315449646,v1:172.21.15.145:6819/3315449646]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:07:27.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:27 smithi172 ceph-mon[22747]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:07:27.513 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:27 smithi145 ceph-mon[26606]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-22T15:07:27.514 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:27 smithi145 ceph-mon[26606]: osdmap e16: 3 total, 2 up, 3 in 2024-04-22T15:07:27.514 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:27 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:07:27.514 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:27 smithi145 ceph-mon[26606]: from='osd.2 [v2:172.21.15.145:6818/3315449646,v1:172.21.15.145:6819/3315449646]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:07:27.514 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:27 smithi145 ceph-mon[26606]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:07:27.514 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:27 smithi145 ceph-mon[21899]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-22T15:07:27.514 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:27 smithi145 ceph-mon[21899]: osdmap e16: 3 total, 2 up, 3 in 2024-04-22T15:07:27.514 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:27 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:07:27.514 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:27 smithi145 ceph-mon[21899]: from='osd.2 [v2:172.21.15.145:6818/3315449646,v1:172.21.15.145:6819/3315449646]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:07:27.515 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:27 smithi145 ceph-mon[21899]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:07:27.515 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:07:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2[35352]: 2024-04-22T15:07:27.206+0000 7fc88848f700 -1 osd.2 0 waiting for initial osdmap 2024-04-22T15:07:27.515 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:07:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2[35352]: 2024-04-22T15:07:27.209+0000 7fc883e28700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T15:07:28.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:28 smithi172 ceph-mon[22747]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-04-22T15:07:28.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:28 smithi172 ceph-mon[22747]: osdmap e17: 3 total, 2 up, 3 in 2024-04-22T15:07:28.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:28 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:07:28.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:28 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:07:28.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:28 smithi172 ceph-mon[22747]: pgmap v71: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:28.594 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:28 smithi145 ceph-mon[21899]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-04-22T15:07:28.594 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:28 smithi145 ceph-mon[21899]: osdmap e17: 3 total, 2 up, 3 in 2024-04-22T15:07:28.594 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:28 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:07:28.594 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:28 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:07:28.594 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:28 smithi145 ceph-mon[21899]: pgmap v71: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:28.594 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:28 smithi145 ceph-mon[26606]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-04-22T15:07:28.595 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:28 smithi145 ceph-mon[26606]: osdmap e17: 3 total, 2 up, 3 in 2024-04-22T15:07:28.595 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:28 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:07:28.595 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:28 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:07:28.595 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:28 smithi145 ceph-mon[26606]: pgmap v71: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T15:07:29.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:29 smithi145 ceph-mon[21899]: purged_snaps scrub starts 2024-04-22T15:07:29.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:29 smithi145 ceph-mon[21899]: purged_snaps scrub ok 2024-04-22T15:07:29.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:29 smithi145 ceph-mon[21899]: osd.2 [v2:172.21.15.145:6818/3315449646,v1:172.21.15.145:6819/3315449646] boot 2024-04-22T15:07:29.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:29 smithi145 ceph-mon[21899]: osdmap e18: 3 total, 3 up, 3 in 2024-04-22T15:07:29.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:29 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:07:29.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:29 smithi145 ceph-mon[26606]: purged_snaps scrub starts 2024-04-22T15:07:29.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:29 smithi145 ceph-mon[26606]: purged_snaps scrub ok 2024-04-22T15:07:29.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:29 smithi145 ceph-mon[26606]: osd.2 [v2:172.21.15.145:6818/3315449646,v1:172.21.15.145:6819/3315449646] boot 2024-04-22T15:07:29.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:29 smithi145 ceph-mon[26606]: osdmap e18: 3 total, 3 up, 3 in 2024-04-22T15:07:29.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:29 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:07:29.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:29 smithi172 ceph-mon[22747]: purged_snaps scrub starts 2024-04-22T15:07:29.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:29 smithi172 ceph-mon[22747]: purged_snaps scrub ok 2024-04-22T15:07:29.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:29 smithi172 ceph-mon[22747]: osd.2 [v2:172.21.15.145:6818/3315449646,v1:172.21.15.145:6819/3315449646] boot 2024-04-22T15:07:29.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:29 smithi172 ceph-mon[22747]: osdmap e18: 3 total, 3 up, 3 in 2024-04-22T15:07:29.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:29 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:07:30.121 INFO:teuthology.orchestra.run.smithi145.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-22T15:07:30.121 INFO:teuthology.orchestra.run.smithi145.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-22T15:07:30.121 INFO:teuthology.orchestra.run.smithi145.stderr: stderr: 10+0 records in 2024-04-22T15:07:30.121 INFO:teuthology.orchestra.run.smithi145.stderr:10+0 records out 2024-04-22T15:07:30.121 INFO:teuthology.orchestra.run.smithi145.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0181635 s, 577 MB/s 2024-04-22T15:07:30.121 INFO:teuthology.orchestra.run.smithi145.stderr:--> Zapping successful for: 2024-04-22T15:07:30.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:30 smithi145 ceph-mon[21899]: osdmap e19: 3 total, 3 up, 3 in 2024-04-22T15:07:30.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:30 smithi145 ceph-mon[21899]: pgmap v74: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:30.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:30 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-22T15:07:30.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:30 smithi145 ceph-mon[26606]: osdmap e19: 3 total, 3 up, 3 in 2024-04-22T15:07:30.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:30 smithi145 ceph-mon[26606]: pgmap v74: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:30.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:30 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-22T15:07:30.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:30 smithi172 ceph-mon[22747]: osdmap e19: 3 total, 3 up, 3 in 2024-04-22T15:07:30.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:30 smithi172 ceph-mon[22747]: pgmap v74: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:30.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:30 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-22T15:07:30.764 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi145:vg_nvme/lv_1 2024-04-22T15:07:31.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:31 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-22T15:07:31.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:31 smithi145 ceph-mon[21899]: osdmap e20: 3 total, 3 up, 3 in 2024-04-22T15:07:31.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:31 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-22T15:07:31.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:31 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-22T15:07:31.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:31 smithi145 ceph-mon[26606]: osdmap e20: 3 total, 3 up, 3 in 2024-04-22T15:07:31.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:31 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-22T15:07:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:31 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-22T15:07:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:31 smithi172 ceph-mon[22747]: osdmap e20: 3 total, 3 up, 3 in 2024-04-22T15:07:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:31 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-22T15:07:32.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:32 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-22T15:07:32.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:32 smithi145 ceph-mon[21899]: osdmap e21: 3 total, 3 up, 3 in 2024-04-22T15:07:32.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:32 smithi145 ceph-mon[21899]: pgmap v77: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:32.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:32 smithi145 ceph-mon[21899]: Detected new or changed devices on smithi145 2024-04-22T15:07:32.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:32 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:32.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:32 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:32.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:32 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:32.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:32 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:32.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:32 smithi145 ceph-mon[21899]: Adjusting osd_memory_target on smithi145 to 5354M 2024-04-22T15:07:32.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:32 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:32.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:32 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:32.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:32 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-22T15:07:32.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:32 smithi145 ceph-mon[26606]: osdmap e21: 3 total, 3 up, 3 in 2024-04-22T15:07:32.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:32 smithi145 ceph-mon[26606]: pgmap v77: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:32.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:32 smithi145 ceph-mon[26606]: Detected new or changed devices on smithi145 2024-04-22T15:07:32.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:32 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:32.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:32 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:32.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:32 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:32.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:32 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:32.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:32 smithi145 ceph-mon[26606]: Adjusting osd_memory_target on smithi145 to 5354M 2024-04-22T15:07:32.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:32 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:32.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:32 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:32.376 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:07:32 smithi145 sudo[36640]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-22T15:07:32.376 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:07:32 smithi145 sudo[36640]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T15:07:32.376 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:07:32 smithi145 sudo[36640]: pam_unix(sudo:session): session closed for user root 2024-04-22T15:07:32.683 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:07:32 smithi145 sudo[36643]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-22T15:07:32.683 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:07:32 smithi145 sudo[36643]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T15:07:32.683 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:07:32 smithi145 sudo[36643]: pam_unix(sudo:session): session closed for user root 2024-04-22T15:07:32.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:32 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-22T15:07:32.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:32 smithi172 ceph-mon[22747]: osdmap e21: 3 total, 3 up, 3 in 2024-04-22T15:07:32.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:32 smithi172 ceph-mon[22747]: pgmap v77: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:32.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:32 smithi172 ceph-mon[22747]: Detected new or changed devices on smithi145 2024-04-22T15:07:32.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:32 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:32.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:32 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:32.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:32 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:32.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:32 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:32.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:32 smithi172 ceph-mon[22747]: Adjusting osd_memory_target on smithi145 to 5354M 2024-04-22T15:07:32.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:32 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:32.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:32 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:32.938 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:07:32 smithi145 sudo[36665]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-22T15:07:32.938 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:07:32 smithi145 sudo[36665]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T15:07:32.939 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:07:32 smithi145 sudo[36665]: pam_unix(sudo:session): session closed for user root 2024-04-22T15:07:33.197 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:07:32 smithi145 sudo[36674]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-22T15:07:33.198 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:07:32 smithi145 sudo[36674]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T15:07:33.198 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:07:32 smithi145 sudo[36674]: pam_unix(sudo:session): session closed for user root 2024-04-22T15:07:33.463 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:33 smithi145 ceph-mon[26606]: osdmap e22: 3 total, 3 up, 3 in 2024-04-22T15:07:33.463 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:33 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:33.464 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:33 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:07:33.464 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:33 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:07:33.464 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:33 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:33.464 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:07:33 smithi145 sudo[36698]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-22T15:07:33.464 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:07:33 smithi145 sudo[36698]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T15:07:33.464 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:07:33 smithi145 sudo[36698]: pam_unix(sudo:session): session closed for user root 2024-04-22T15:07:33.464 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:33 smithi145 ceph-mon[21899]: osdmap e22: 3 total, 3 up, 3 in 2024-04-22T15:07:33.464 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:33 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:33.465 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:33 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:07:33.465 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:33 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:07:33.465 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:33 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:33.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:33 smithi172 ceph-mon[22747]: osdmap e22: 3 total, 3 up, 3 in 2024-04-22T15:07:33.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:33 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:33.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:33 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:07:33.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:33 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:07:33.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:33 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:33.724 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:07:33 smithi145 sudo[36757]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-22T15:07:33.725 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:07:33 smithi145 sudo[36757]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T15:07:33.725 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:07:33 smithi145 sudo[36757]: pam_unix(sudo:session): session closed for user root 2024-04-22T15:07:33.725 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:33 smithi145 sudo[36781]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-22T15:07:33.725 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:33 smithi145 sudo[36781]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T15:07:33.725 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:33 smithi145 sudo[36781]: pam_unix(sudo:session): session closed for user root 2024-04-22T15:07:34.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:33 smithi145 sudo[36802]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-22T15:07:34.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:33 smithi145 sudo[36802]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T15:07:34.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:33 smithi145 sudo[36802]: pam_unix(sudo:session): session closed for user root 2024-04-22T15:07:34.477 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:34 smithi145 sudo[36808]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-22T15:07:34.477 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:34 smithi145 sudo[36808]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T15:07:34.477 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:34 smithi145 sudo[36808]: pam_unix(sudo:session): session closed for user root 2024-04-22T15:07:34.796 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:34 smithi145 sudo[36812]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-22T15:07:34.796 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:34 smithi145 sudo[36812]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T15:07:34.796 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:34 smithi145 sudo[36812]: pam_unix(sudo:session): session closed for user root 2024-04-22T15:07:34.975 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:34 smithi172 sudo[24270]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-22T15:07:34.975 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:34 smithi172 sudo[24270]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T15:07:34.975 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:34 smithi172 sudo[24270]: pam_unix(sudo:session): session closed for user root 2024-04-22T15:07:35.099 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:34 smithi145 ceph-mon[21899]: from='client.14307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:07:35.099 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:34 smithi145 ceph-mon[21899]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T15:07:35.100 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:34 smithi145 ceph-mon[21899]: pgmap v79: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:35.100 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:34 smithi145 ceph-mon[21899]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T15:07:35.100 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:34 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:07:35.100 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:34 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:07:35.100 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:34 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:07:35.100 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:34 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:07:35.100 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:34 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:07:35.100 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:34 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:07:35.100 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:34 smithi145 ceph-mon[26606]: from='client.14307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:07:35.100 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:34 smithi145 ceph-mon[26606]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T15:07:35.100 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:34 smithi145 ceph-mon[26606]: pgmap v79: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:35.101 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:34 smithi145 ceph-mon[26606]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T15:07:35.101 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:34 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:07:35.101 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:34 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:07:35.101 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:34 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:07:35.101 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:34 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:07:35.101 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:34 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:07:35.101 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:34 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:07:35.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:34 smithi172 sudo[24273]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-22T15:07:35.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:34 smithi172 sudo[24273]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T15:07:35.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:34 smithi172 sudo[24273]: pam_unix(sudo:session): session closed for user root 2024-04-22T15:07:35.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:34 smithi172 ceph-mon[22747]: from='client.14307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:07:35.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:34 smithi172 ceph-mon[22747]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T15:07:35.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:34 smithi172 ceph-mon[22747]: pgmap v79: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:35.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:34 smithi172 ceph-mon[22747]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T15:07:35.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:34 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:07:35.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:34 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:07:35.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:34 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:07:35.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:34 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:07:35.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:34 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:07:35.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:34 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:07:36.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:35 smithi145 ceph-mon[21899]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T15:07:36.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:35 smithi145 ceph-mon[21899]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T15:07:36.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:35 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:07:36.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:35 smithi145 ceph-mon[21899]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T15:07:36.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:35 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:07:36.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:35 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:07:36.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:35 smithi145 ceph-mon[21899]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T15:07:36.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:35 smithi145 ceph-mon[21899]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-22T15:07:36.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:35 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:36.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:35 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/1530736258' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "25f5ce78-1153-489a-bdc9-7cbd4ff4cfe5"}]: dispatch 2024-04-22T15:07:36.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:35 smithi145 ceph-mon[21899]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "25f5ce78-1153-489a-bdc9-7cbd4ff4cfe5"}]: dispatch 2024-04-22T15:07:36.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:35 smithi145 ceph-mon[21899]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "25f5ce78-1153-489a-bdc9-7cbd4ff4cfe5"}]': finished 2024-04-22T15:07:36.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:35 smithi145 ceph-mon[21899]: osdmap e23: 4 total, 3 up, 4 in 2024-04-22T15:07:36.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:35 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:07:36.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:35 smithi145 ceph-mon[26606]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T15:07:36.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:35 smithi145 ceph-mon[26606]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T15:07:36.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:35 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:07:36.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:35 smithi145 ceph-mon[26606]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T15:07:36.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:35 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:07:36.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:35 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:07:36.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:35 smithi145 ceph-mon[26606]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T15:07:36.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:35 smithi145 ceph-mon[26606]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-22T15:07:36.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:35 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:36.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:35 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/1530736258' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "25f5ce78-1153-489a-bdc9-7cbd4ff4cfe5"}]: dispatch 2024-04-22T15:07:36.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:35 smithi145 ceph-mon[26606]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "25f5ce78-1153-489a-bdc9-7cbd4ff4cfe5"}]: dispatch 2024-04-22T15:07:36.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:35 smithi145 ceph-mon[26606]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "25f5ce78-1153-489a-bdc9-7cbd4ff4cfe5"}]': finished 2024-04-22T15:07:36.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:35 smithi145 ceph-mon[26606]: osdmap e23: 4 total, 3 up, 4 in 2024-04-22T15:07:36.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:35 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:07:36.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:35 smithi172 ceph-mon[22747]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T15:07:36.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:35 smithi172 ceph-mon[22747]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T15:07:36.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:35 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:07:36.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:35 smithi172 ceph-mon[22747]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T15:07:36.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:35 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:07:36.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:35 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:07:36.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:35 smithi172 ceph-mon[22747]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T15:07:36.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:35 smithi172 ceph-mon[22747]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-22T15:07:36.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:35 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:36.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:35 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/1530736258' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "25f5ce78-1153-489a-bdc9-7cbd4ff4cfe5"}]: dispatch 2024-04-22T15:07:36.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:35 smithi172 ceph-mon[22747]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "25f5ce78-1153-489a-bdc9-7cbd4ff4cfe5"}]: dispatch 2024-04-22T15:07:36.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:35 smithi172 ceph-mon[22747]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "25f5ce78-1153-489a-bdc9-7cbd4ff4cfe5"}]': finished 2024-04-22T15:07:36.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:35 smithi172 ceph-mon[22747]: osdmap e23: 4 total, 3 up, 4 in 2024-04-22T15:07:36.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:35 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:07:37.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:36 smithi145 ceph-mon[21899]: pgmap v81: 1 pgs: 1 creating+peering; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:37.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:36 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2139566571' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:07:37.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:36 smithi145 ceph-mon[26606]: pgmap v81: 1 pgs: 1 creating+peering; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:37.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:36 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2139566571' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:07:37.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:36 smithi172 ceph-mon[22747]: pgmap v81: 1 pgs: 1 creating+peering; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:37.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:36 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2139566571' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:07:39.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:38 smithi145 ceph-mon[21899]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:39.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:38 smithi145 ceph-mon[26606]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:39.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:38 smithi172 ceph-mon[22747]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:41.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:40 smithi145 ceph-mon[21899]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:41.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:40 smithi145 ceph-mon[26606]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:41.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:40 smithi172 ceph-mon[22747]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:43.358 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:42 smithi145 ceph-mon[21899]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:43.358 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:42 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T15:07:43.358 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:42 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:43.358 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:42 smithi145 ceph-mon[26606]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:43.358 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:42 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T15:07:43.359 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:42 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:43.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:42 smithi172 ceph-mon[22747]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:43.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:42 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T15:07:43.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:42 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:44.291 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:43 smithi145 ceph-mon[21899]: Deploying daemon osd.3 on smithi145 2024-04-22T15:07:44.291 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:43 smithi145 ceph-mon[26606]: Deploying daemon osd.3 on smithi145 2024-04-22T15:07:44.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:43 smithi172 ceph-mon[22747]: Deploying daemon osd.3 on smithi145 2024-04-22T15:07:45.251 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:44 smithi145 ceph-mon[21899]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:45.252 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:44 smithi145 ceph-mon[26606]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:45.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:44 smithi172 ceph-mon[22747]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:47.298 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:47 smithi145 ceph-mon[21899]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:47.299 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:47 smithi145 ceph-mon[26606]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:47.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:47 smithi172 ceph-mon[22747]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:48.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:48 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:48.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:48 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:07:48.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:48 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:48.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:48 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:07:48.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:48 smithi145 ceph-mon[21899]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:48.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:48 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:48.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:48 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:07:48.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:48 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:48.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:48 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:07:48.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:48 smithi145 ceph-mon[26606]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:48.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:48 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:48.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:48 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:07:48.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:48 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:48.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:48 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:07:48.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:48 smithi172 ceph-mon[22747]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:51.131 INFO:teuthology.orchestra.run.smithi145.stdout:Created osd(s) 3 on host 'smithi145' 2024-04-22T15:07:51.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:50 smithi145 ceph-mon[21899]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:51.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:50 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:51.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:50 smithi145 ceph-mon[21899]: from='osd.3 [v2:172.21.15.145:6826/922673347,v1:172.21.15.145:6827/922673347]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-22T15:07:51.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:50 smithi145 ceph-mon[21899]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-22T15:07:51.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:50 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:51.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:50 smithi145 ceph-mon[26606]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:51.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:50 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:51.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:50 smithi145 ceph-mon[26606]: from='osd.3 [v2:172.21.15.145:6826/922673347,v1:172.21.15.145:6827/922673347]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-22T15:07:51.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:50 smithi145 ceph-mon[26606]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-22T15:07:51.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:50 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:51.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:50 smithi172 ceph-mon[22747]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:51.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:50 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:51.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:50 smithi172 ceph-mon[22747]: from='osd.3 [v2:172.21.15.145:6826/922673347,v1:172.21.15.145:6827/922673347]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-22T15:07:51.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:50 smithi172 ceph-mon[22747]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-22T15:07:51.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:50 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:51.803 DEBUG:teuthology.orchestra.run.smithi145:osd.3> sudo journalctl -f -n 0 -u ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.3.service 2024-04-22T15:07:51.806 INFO:tasks.cephadm:Deploying osd.4 on smithi172 with /dev/vg_nvme/lv_4... 2024-04-22T15:07:51.806 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-22T15:07:52.092 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:07:52.092 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:52.092 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:52.092 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:07:52.092 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:51 smithi145 ceph-mon[26606]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-22T15:07:52.092 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:51 smithi145 ceph-mon[26606]: osdmap e24: 4 total, 3 up, 4 in 2024-04-22T15:07:52.092 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:51 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:07:52.092 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:51 smithi145 ceph-mon[26606]: from='osd.3 [v2:172.21.15.145:6826/922673347,v1:172.21.15.145:6827/922673347]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:07:52.092 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:51 smithi145 ceph-mon[26606]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:07:52.093 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:51 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:07:52.093 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:51 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:52.093 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:51 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:52.093 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:51 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:07:52.093 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:51 smithi145 ceph-mon[21899]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-22T15:07:52.093 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:51 smithi145 ceph-mon[21899]: osdmap e24: 4 total, 3 up, 4 in 2024-04-22T15:07:52.093 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:51 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:07:52.093 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:51 smithi145 ceph-mon[21899]: from='osd.3 [v2:172.21.15.145:6826/922673347,v1:172.21.15.145:6827/922673347]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:07:52.093 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:51 smithi145 ceph-mon[21899]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:07:52.129 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:51 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:07:52.130 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:51 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:52.130 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:51 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:52.130 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:51 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:07:52.130 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:51 smithi172 ceph-mon[22747]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-22T15:07:52.130 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:51 smithi172 ceph-mon[22747]: osdmap e24: 4 total, 3 up, 4 in 2024-04-22T15:07:52.130 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:51 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:07:52.130 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:51 smithi172 ceph-mon[22747]: from='osd.3 [v2:172.21.15.145:6826/922673347,v1:172.21.15.145:6827/922673347]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:07:52.130 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:51 smithi172 ceph-mon[22747]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:07:52.623 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:07:52 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3[38329]: 2024-04-22T15:07:52.358+0000 7f658440f700 -1 osd.3 0 waiting for initial osdmap 2024-04-22T15:07:52.623 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:07:52 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3[38329]: 2024-04-22T15:07:52.362+0000 7f657eda6700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T15:07:53.211 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:52 smithi145 ceph-mon[26606]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:53.211 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:52 smithi145 ceph-mon[26606]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-04-22T15:07:53.211 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:52 smithi145 ceph-mon[26606]: osdmap e25: 4 total, 3 up, 4 in 2024-04-22T15:07:53.211 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:52 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:07:53.212 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:52 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:07:53.212 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:52 smithi145 ceph-mon[21899]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:53.212 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:52 smithi145 ceph-mon[21899]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-04-22T15:07:53.212 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:52 smithi145 ceph-mon[21899]: osdmap e25: 4 total, 3 up, 4 in 2024-04-22T15:07:53.212 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:52 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:07:53.212 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:52 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:07:53.305 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:52 smithi172 ceph-mon[22747]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:53.306 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:52 smithi172 ceph-mon[22747]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-04-22T15:07:53.306 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:52 smithi172 ceph-mon[22747]: osdmap e25: 4 total, 3 up, 4 in 2024-04-22T15:07:53.306 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:52 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:07:53.306 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:52 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:07:53.679 INFO:teuthology.orchestra.run.smithi172.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-22T15:07:53.679 INFO:teuthology.orchestra.run.smithi172.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-22T15:07:53.679 INFO:teuthology.orchestra.run.smithi172.stderr: stderr: 10+0 records in 2024-04-22T15:07:53.680 INFO:teuthology.orchestra.run.smithi172.stderr:10+0 records out 2024-04-22T15:07:53.680 INFO:teuthology.orchestra.run.smithi172.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0125613 s, 835 MB/s 2024-04-22T15:07:53.680 INFO:teuthology.orchestra.run.smithi172.stderr:--> Zapping successful for: 2024-04-22T15:07:54.249 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi172:vg_nvme/lv_4 2024-04-22T15:07:54.558 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:54 smithi172 ceph-mon[22747]: purged_snaps scrub starts 2024-04-22T15:07:54.558 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:54 smithi172 ceph-mon[22747]: purged_snaps scrub ok 2024-04-22T15:07:54.558 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:54 smithi172 ceph-mon[22747]: osd.3 [v2:172.21.15.145:6826/922673347,v1:172.21.15.145:6827/922673347] boot 2024-04-22T15:07:54.558 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:54 smithi172 ceph-mon[22747]: osdmap e26: 4 total, 4 up, 4 in 2024-04-22T15:07:54.558 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:54 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:07:54.558 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:54 smithi172 ceph-mon[22747]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:54.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:54 smithi145 ceph-mon[21899]: purged_snaps scrub starts 2024-04-22T15:07:54.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:54 smithi145 ceph-mon[21899]: purged_snaps scrub ok 2024-04-22T15:07:54.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:54 smithi145 ceph-mon[21899]: osd.3 [v2:172.21.15.145:6826/922673347,v1:172.21.15.145:6827/922673347] boot 2024-04-22T15:07:54.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:54 smithi145 ceph-mon[21899]: osdmap e26: 4 total, 4 up, 4 in 2024-04-22T15:07:54.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:54 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:07:54.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:54 smithi145 ceph-mon[21899]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:54.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:54 smithi145 ceph-mon[26606]: purged_snaps scrub starts 2024-04-22T15:07:54.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:54 smithi145 ceph-mon[26606]: purged_snaps scrub ok 2024-04-22T15:07:54.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:54 smithi145 ceph-mon[26606]: osd.3 [v2:172.21.15.145:6826/922673347,v1:172.21.15.145:6827/922673347] boot 2024-04-22T15:07:54.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:54 smithi145 ceph-mon[26606]: osdmap e26: 4 total, 4 up, 4 in 2024-04-22T15:07:54.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:54 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:07:54.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:54 smithi145 ceph-mon[26606]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T15:07:55.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:55 smithi145 ceph-mon[21899]: osdmap e27: 4 total, 4 up, 4 in 2024-04-22T15:07:55.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:55 smithi145 ceph-mon[21899]: Detected new or changed devices on smithi145 2024-04-22T15:07:55.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:55 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:55.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:55 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:55.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:55 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:55.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:55 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:55.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:55 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:55.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:55 smithi145 ceph-mon[21899]: Adjusting osd_memory_target on smithi145 to 4016M 2024-04-22T15:07:55.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:55 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:55.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:55 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:55.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:55 smithi145 ceph-mon[26606]: osdmap e27: 4 total, 4 up, 4 in 2024-04-22T15:07:55.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:55 smithi145 ceph-mon[26606]: Detected new or changed devices on smithi145 2024-04-22T15:07:55.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:55 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:55.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:55 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:55.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:55 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:55.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:55 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:55.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:55 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:55.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:55 smithi145 ceph-mon[26606]: Adjusting osd_memory_target on smithi145 to 4016M 2024-04-22T15:07:55.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:55 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:55.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:55 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:55.711 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:55 smithi172 ceph-mon[22747]: osdmap e27: 4 total, 4 up, 4 in 2024-04-22T15:07:55.711 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:55 smithi172 ceph-mon[22747]: Detected new or changed devices on smithi145 2024-04-22T15:07:55.711 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:55 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:55.711 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:55 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:55.711 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:55 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:55.711 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:55 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:55.711 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:55 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:07:55.711 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:55 smithi172 ceph-mon[22747]: Adjusting osd_memory_target on smithi145 to 4016M 2024-04-22T15:07:55.711 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:55 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:55.711 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:55 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:07:56.459 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:56 smithi172 ceph-mon[22747]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:07:56.459 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:56 smithi172 ceph-mon[22747]: from='client.24209 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi172:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:07:56.459 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:56 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:07:56.459 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:56 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:07:56.460 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:56 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:56.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:56 smithi145 ceph-mon[26606]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:07:56.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:56 smithi145 ceph-mon[26606]: from='client.24209 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi172:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:07:56.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:56 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:07:56.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:56 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:07:56.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:56 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:56.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:56 smithi145 ceph-mon[21899]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:07:56.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:56 smithi145 ceph-mon[21899]: from='client.24209 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi172:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:07:56.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:56 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:07:56.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:56 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:07:56.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:56 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:07:59.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:58 smithi172 ceph-mon[22747]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:07:59.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:58 smithi172 ceph-mon[22747]: from='client.? 172.21.15.172:0/4146786515' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eefb46ad-39a3-40bf-a966-65f7a32841f0"}]: dispatch 2024-04-22T15:07:59.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:58 smithi172 ceph-mon[22747]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eefb46ad-39a3-40bf-a966-65f7a32841f0"}]: dispatch 2024-04-22T15:07:59.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:58 smithi172 ceph-mon[22747]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "eefb46ad-39a3-40bf-a966-65f7a32841f0"}]': finished 2024-04-22T15:07:59.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:58 smithi172 ceph-mon[22747]: osdmap e28: 5 total, 4 up, 5 in 2024-04-22T15:07:59.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:58 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:07:59.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:07:58 smithi172 ceph-mon[22747]: from='client.? 172.21.15.172:0/157785773' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:07:59.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:58 smithi145 ceph-mon[21899]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:07:59.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:58 smithi145 ceph-mon[21899]: from='client.? 172.21.15.172:0/4146786515' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eefb46ad-39a3-40bf-a966-65f7a32841f0"}]: dispatch 2024-04-22T15:07:59.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:58 smithi145 ceph-mon[21899]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eefb46ad-39a3-40bf-a966-65f7a32841f0"}]: dispatch 2024-04-22T15:07:59.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:58 smithi145 ceph-mon[21899]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "eefb46ad-39a3-40bf-a966-65f7a32841f0"}]': finished 2024-04-22T15:07:59.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:58 smithi145 ceph-mon[21899]: osdmap e28: 5 total, 4 up, 5 in 2024-04-22T15:07:59.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:58 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:07:59.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:07:58 smithi145 ceph-mon[21899]: from='client.? 172.21.15.172:0/157785773' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:07:59.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:58 smithi145 ceph-mon[26606]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:07:59.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:58 smithi145 ceph-mon[26606]: from='client.? 172.21.15.172:0/4146786515' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eefb46ad-39a3-40bf-a966-65f7a32841f0"}]: dispatch 2024-04-22T15:07:59.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:58 smithi145 ceph-mon[26606]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eefb46ad-39a3-40bf-a966-65f7a32841f0"}]: dispatch 2024-04-22T15:07:59.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:58 smithi145 ceph-mon[26606]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "eefb46ad-39a3-40bf-a966-65f7a32841f0"}]': finished 2024-04-22T15:07:59.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:58 smithi145 ceph-mon[26606]: osdmap e28: 5 total, 4 up, 5 in 2024-04-22T15:07:59.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:58 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:07:59.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:07:58 smithi145 ceph-mon[26606]: from='client.? 172.21.15.172:0/157785773' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:08:01.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:00 smithi145 ceph-mon[21899]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:01.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:00 smithi145 ceph-mon[26606]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:01.414 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:00 smithi172 ceph-mon[22747]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:03.169 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:02 smithi172 ceph-mon[22747]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:03.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:02 smithi145 ceph-mon[21899]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:03.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:02 smithi145 ceph-mon[26606]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:05.081 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:04 smithi172 ceph-mon[22747]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:05.081 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:04 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T15:08:05.081 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:04 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:05.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:04 smithi145 ceph-mon[21899]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:05.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:04 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T15:08:05.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:04 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:05.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:04 smithi145 ceph-mon[26606]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:05.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:04 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T15:08:05.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:04 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:06.137 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:06 smithi172 ceph-mon[22747]: Deploying daemon osd.4 on smithi172 2024-04-22T15:08:06.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:06 smithi145 ceph-mon[21899]: Deploying daemon osd.4 on smithi172 2024-04-22T15:08:06.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:06 smithi145 ceph-mon[26606]: Deploying daemon osd.4 on smithi172 2024-04-22T15:08:07.201 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:07 smithi172 ceph-mon[22747]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:07.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:07 smithi145 ceph-mon[21899]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:07.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:07 smithi145 ceph-mon[26606]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:09.090 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:09 smithi172 ceph-mon[22747]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:09.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:09 smithi145 ceph-mon[21899]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:09.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:09 smithi145 ceph-mon[26606]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:10.431 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:10 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:10.431 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:10 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:08:10.431 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:10 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:10.431 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:10 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:08:10.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:10 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:10.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:10 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:08:10.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:10 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:10.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:10 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:08:10.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:10 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:10.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:10 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:08:10.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:10 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:10.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:10 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:08:11.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:11 smithi172 ceph-mon[22747]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:11.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:11 smithi145 ceph-mon[21899]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:11.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:11 smithi145 ceph-mon[26606]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:12.095 INFO:teuthology.orchestra.run.smithi172.stdout:Created osd(s) 4 on host 'smithi172' 2024-04-22T15:08:12.688 DEBUG:teuthology.orchestra.run.smithi172:osd.4> sudo journalctl -f -n 0 -u ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.4.service 2024-04-22T15:08:12.690 INFO:tasks.cephadm:Deploying osd.5 on smithi172 with /dev/vg_nvme/lv_3... 2024-04-22T15:08:12.691 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-22T15:08:12.916 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:12 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:12.916 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:12 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:12.916 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:12 smithi172 ceph-mon[22747]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:12.916 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:12 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:12.916 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:12 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:08:12.916 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:12 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:12.916 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:12 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:08:12.916 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:12 smithi172 ceph-mon[22747]: from='osd.4 [v2:172.21.15.172:6800/3136053699,v1:172.21.15.172:6801/3136053699]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T15:08:12.916 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:12 smithi172 ceph-mon[22747]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T15:08:13.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:12 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:13.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:12 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:13.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:12 smithi145 ceph-mon[21899]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:13.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:12 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:13.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:12 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:08:13.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:12 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:13.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:12 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:08:13.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:12 smithi145 ceph-mon[21899]: from='osd.4 [v2:172.21.15.172:6800/3136053699,v1:172.21.15.172:6801/3136053699]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T15:08:13.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:12 smithi145 ceph-mon[21899]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T15:08:13.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:12 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:13.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:12 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:13.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:12 smithi145 ceph-mon[26606]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:13.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:12 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:13.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:12 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:08:13.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:12 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:13.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:12 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:08:13.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:12 smithi145 ceph-mon[26606]: from='osd.4 [v2:172.21.15.172:6800/3136053699,v1:172.21.15.172:6801/3136053699]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T15:08:13.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:12 smithi145 ceph-mon[26606]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T15:08:14.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:14 smithi145 ceph-mon[21899]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-22T15:08:14.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:14 smithi145 ceph-mon[21899]: osdmap e29: 5 total, 4 up, 5 in 2024-04-22T15:08:14.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:14 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:08:14.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:14 smithi145 ceph-mon[21899]: from='osd.4 [v2:172.21.15.172:6800/3136053699,v1:172.21.15.172:6801/3136053699]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:08:14.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:14 smithi145 ceph-mon[21899]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:08:14.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:14 smithi145 ceph-mon[26606]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-22T15:08:14.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:14 smithi145 ceph-mon[26606]: osdmap e29: 5 total, 4 up, 5 in 2024-04-22T15:08:14.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:14 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:08:14.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:14 smithi145 ceph-mon[26606]: from='osd.4 [v2:172.21.15.172:6800/3136053699,v1:172.21.15.172:6801/3136053699]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:08:14.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:14 smithi145 ceph-mon[26606]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:08:14.429 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:14 smithi172 ceph-mon[22747]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-22T15:08:14.429 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:14 smithi172 ceph-mon[22747]: osdmap e29: 5 total, 4 up, 5 in 2024-04-22T15:08:14.429 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:14 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:08:14.429 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:14 smithi172 ceph-mon[22747]: from='osd.4 [v2:172.21.15.172:6800/3136053699,v1:172.21.15.172:6801/3136053699]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:08:14.429 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:14 smithi172 ceph-mon[22747]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:08:14.430 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:08:14 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4[26200]: 2024-04-22T15:08:14.116+0000 7fb558cb6700 -1 osd.4 0 waiting for initial osdmap 2024-04-22T15:08:14.430 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:08:14 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4[26200]: 2024-04-22T15:08:14.121+0000 7fb551649700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T15:08:15.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:15 smithi145 ceph-mon[21899]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:15.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:15 smithi145 ceph-mon[21899]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]': finished 2024-04-22T15:08:15.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:15 smithi145 ceph-mon[21899]: osdmap e30: 5 total, 4 up, 5 in 2024-04-22T15:08:15.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:15 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:08:15.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:15 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:08:15.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:15 smithi145 ceph-mon[26606]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:15.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:15 smithi145 ceph-mon[26606]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]': finished 2024-04-22T15:08:15.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:15 smithi145 ceph-mon[26606]: osdmap e30: 5 total, 4 up, 5 in 2024-04-22T15:08:15.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:15 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:08:15.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:15 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:08:15.432 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:15 smithi172 ceph-mon[22747]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T15:08:15.432 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:15 smithi172 ceph-mon[22747]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]': finished 2024-04-22T15:08:15.432 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:15 smithi172 ceph-mon[22747]: osdmap e30: 5 total, 4 up, 5 in 2024-04-22T15:08:15.432 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:15 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:08:15.432 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:15 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:08:16.105 INFO:teuthology.orchestra.run.smithi172.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-22T15:08:16.105 INFO:teuthology.orchestra.run.smithi172.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-22T15:08:16.105 INFO:teuthology.orchestra.run.smithi172.stderr: stderr: 10+0 records in 2024-04-22T15:08:16.105 INFO:teuthology.orchestra.run.smithi172.stderr:10+0 records out 2024-04-22T15:08:16.105 INFO:teuthology.orchestra.run.smithi172.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0121968 s, 860 MB/s 2024-04-22T15:08:16.105 INFO:teuthology.orchestra.run.smithi172.stderr:--> Zapping successful for: 2024-04-22T15:08:16.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:16 smithi172 ceph-mon[22747]: purged_snaps scrub starts 2024-04-22T15:08:16.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:16 smithi172 ceph-mon[22747]: purged_snaps scrub ok 2024-04-22T15:08:16.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:16 smithi172 ceph-mon[22747]: osd.4 [v2:172.21.15.172:6800/3136053699,v1:172.21.15.172:6801/3136053699] boot 2024-04-22T15:08:16.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:16 smithi172 ceph-mon[22747]: osdmap e31: 5 total, 5 up, 5 in 2024-04-22T15:08:16.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:16 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:08:16.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:16 smithi145 ceph-mon[21899]: purged_snaps scrub starts 2024-04-22T15:08:16.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:16 smithi145 ceph-mon[21899]: purged_snaps scrub ok 2024-04-22T15:08:16.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:16 smithi145 ceph-mon[21899]: osd.4 [v2:172.21.15.172:6800/3136053699,v1:172.21.15.172:6801/3136053699] boot 2024-04-22T15:08:16.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:16 smithi145 ceph-mon[21899]: osdmap e31: 5 total, 5 up, 5 in 2024-04-22T15:08:16.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:16 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:08:16.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:16 smithi145 ceph-mon[26606]: purged_snaps scrub starts 2024-04-22T15:08:16.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:16 smithi145 ceph-mon[26606]: purged_snaps scrub ok 2024-04-22T15:08:16.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:16 smithi145 ceph-mon[26606]: osd.4 [v2:172.21.15.172:6800/3136053699,v1:172.21.15.172:6801/3136053699] boot 2024-04-22T15:08:16.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:16 smithi145 ceph-mon[26606]: osdmap e31: 5 total, 5 up, 5 in 2024-04-22T15:08:16.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:16 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:08:16.851 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi172:vg_nvme/lv_3 2024-04-22T15:08:17.389 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:17 smithi172 ceph-mon[22747]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-22T15:08:17.389 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:17 smithi172 ceph-mon[22747]: osdmap e32: 5 total, 5 up, 5 in 2024-04-22T15:08:17.389 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:17 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:17.389 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:17 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:08:17.389 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:17 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:17.389 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:17 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:17.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:17 smithi145 ceph-mon[21899]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-22T15:08:17.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:17 smithi145 ceph-mon[21899]: osdmap e32: 5 total, 5 up, 5 in 2024-04-22T15:08:17.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:17 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:17.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:17 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:08:17.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:17 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:17.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:17 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:17.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:17 smithi145 ceph-mon[26606]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-22T15:08:17.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:17 smithi145 ceph-mon[26606]: osdmap e32: 5 total, 5 up, 5 in 2024-04-22T15:08:17.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:17 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:17.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:17 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:08:17.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:17 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:17.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:17 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:18.348 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:18 smithi172 ceph-mon[22747]: Detected new or changed devices on smithi172 2024-04-22T15:08:18.348 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:18 smithi172 ceph-mon[22747]: Adjusting osd_memory_target on smithi172 to 17088M 2024-04-22T15:08:18.349 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:18 smithi172 ceph-mon[22747]: osdmap e33: 5 total, 5 up, 5 in 2024-04-22T15:08:18.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:18 smithi145 ceph-mon[21899]: Detected new or changed devices on smithi172 2024-04-22T15:08:18.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:18 smithi145 ceph-mon[21899]: Adjusting osd_memory_target on smithi172 to 17088M 2024-04-22T15:08:18.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:18 smithi145 ceph-mon[21899]: osdmap e33: 5 total, 5 up, 5 in 2024-04-22T15:08:18.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:18 smithi145 ceph-mon[26606]: Detected new or changed devices on smithi172 2024-04-22T15:08:18.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:18 smithi145 ceph-mon[26606]: Adjusting osd_memory_target on smithi172 to 17088M 2024-04-22T15:08:18.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:18 smithi145 ceph-mon[26606]: osdmap e33: 5 total, 5 up, 5 in 2024-04-22T15:08:19.339 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:19 smithi172 ceph-mon[22747]: pgmap v112: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-22T15:08:19.340 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:19 smithi172 ceph-mon[22747]: osdmap e34: 5 total, 5 up, 5 in 2024-04-22T15:08:19.340 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:19 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:08:19.340 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:19 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:08:19.340 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:19 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:19.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:19 smithi145 ceph-mon[21899]: pgmap v112: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-22T15:08:19.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:19 smithi145 ceph-mon[21899]: osdmap e34: 5 total, 5 up, 5 in 2024-04-22T15:08:19.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:19 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:08:19.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:19 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:08:19.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:19 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:19.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:19 smithi145 ceph-mon[26606]: pgmap v112: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-22T15:08:19.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:19 smithi145 ceph-mon[26606]: osdmap e34: 5 total, 5 up, 5 in 2024-04-22T15:08:19.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:19 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:08:19.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:19 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:08:19.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:19 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:20.233 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:20 smithi172 ceph-mon[22747]: from='client.24236 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi172:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:08:20.234 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:20 smithi172 ceph-mon[22747]: osdmap e35: 5 total, 5 up, 5 in 2024-04-22T15:08:20.234 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:20 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:08:20.234 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:20 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:08:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:20 smithi145 ceph-mon[21899]: from='client.24236 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi172:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:08:20.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:20 smithi145 ceph-mon[21899]: osdmap e35: 5 total, 5 up, 5 in 2024-04-22T15:08:20.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:20 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:08:20.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:20 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:08:20.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:20 smithi145 ceph-mon[26606]: from='client.24236 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi172:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:08:20.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:20 smithi145 ceph-mon[26606]: osdmap e35: 5 total, 5 up, 5 in 2024-04-22T15:08:20.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:20 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:08:20.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:20 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:08:21.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:21 smithi172 ceph-mon[22747]: pgmap v115: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T15:08:21.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:21 smithi172 ceph-mon[22747]: from='client.? 172.21.15.172:0/4070312073' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "58c9f92d-ff6c-4774-a08a-e2377ea0e95f"}]: dispatch 2024-04-22T15:08:21.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:21 smithi172 ceph-mon[22747]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "58c9f92d-ff6c-4774-a08a-e2377ea0e95f"}]: dispatch 2024-04-22T15:08:21.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:21 smithi172 ceph-mon[22747]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "58c9f92d-ff6c-4774-a08a-e2377ea0e95f"}]': finished 2024-04-22T15:08:21.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:21 smithi172 ceph-mon[22747]: osdmap e36: 6 total, 5 up, 6 in 2024-04-22T15:08:21.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:21 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:08:21.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:21 smithi145 ceph-mon[21899]: pgmap v115: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T15:08:21.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:21 smithi145 ceph-mon[21899]: from='client.? 172.21.15.172:0/4070312073' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "58c9f92d-ff6c-4774-a08a-e2377ea0e95f"}]: dispatch 2024-04-22T15:08:21.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:21 smithi145 ceph-mon[21899]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "58c9f92d-ff6c-4774-a08a-e2377ea0e95f"}]: dispatch 2024-04-22T15:08:21.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:21 smithi145 ceph-mon[21899]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "58c9f92d-ff6c-4774-a08a-e2377ea0e95f"}]': finished 2024-04-22T15:08:21.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:21 smithi145 ceph-mon[21899]: osdmap e36: 6 total, 5 up, 6 in 2024-04-22T15:08:21.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:21 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:08:21.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:21 smithi145 ceph-mon[26606]: pgmap v115: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T15:08:21.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:21 smithi145 ceph-mon[26606]: from='client.? 172.21.15.172:0/4070312073' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "58c9f92d-ff6c-4774-a08a-e2377ea0e95f"}]: dispatch 2024-04-22T15:08:21.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:21 smithi145 ceph-mon[26606]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "58c9f92d-ff6c-4774-a08a-e2377ea0e95f"}]: dispatch 2024-04-22T15:08:21.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:21 smithi145 ceph-mon[26606]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "58c9f92d-ff6c-4774-a08a-e2377ea0e95f"}]': finished 2024-04-22T15:08:21.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:21 smithi145 ceph-mon[26606]: osdmap e36: 6 total, 5 up, 6 in 2024-04-22T15:08:21.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:21 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:08:22.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:22 smithi172 ceph-mon[22747]: from='client.? 172.21.15.172:0/1384752101' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:08:22.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:22 smithi145 ceph-mon[21899]: from='client.? 172.21.15.172:0/1384752101' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:08:22.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:22 smithi145 ceph-mon[26606]: from='client.? 172.21.15.172:0/1384752101' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:08:23.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:23 smithi172 ceph-mon[22747]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 108 KiB/s, 0 objects/s recovering 2024-04-22T15:08:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:23 smithi145 ceph-mon[21899]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 108 KiB/s, 0 objects/s recovering 2024-04-22T15:08:23.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:23 smithi145 ceph-mon[26606]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 108 KiB/s, 0 objects/s recovering 2024-04-22T15:08:25.432 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:25 smithi172 ceph-mon[22747]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-22T15:08:25.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:25 smithi145 ceph-mon[21899]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-22T15:08:25.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:25 smithi145 ceph-mon[26606]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-22T15:08:27.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:27 smithi172 ceph-mon[22747]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-04-22T15:08:27.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:27 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T15:08:27.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:27 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:27.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:27 smithi145 ceph-mon[21899]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-04-22T15:08:27.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:27 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T15:08:27.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:27 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:27.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:27 smithi145 ceph-mon[26606]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-04-22T15:08:27.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:27 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T15:08:27.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:27 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:28.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:28 smithi172 ceph-mon[22747]: Deploying daemon osd.5 on smithi172 2024-04-22T15:08:28.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:28 smithi145 ceph-mon[21899]: Deploying daemon osd.5 on smithi172 2024-04-22T15:08:28.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:28 smithi145 ceph-mon[26606]: Deploying daemon osd.5 on smithi172 2024-04-22T15:08:29.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:29 smithi172 ceph-mon[22747]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-04-22T15:08:29.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:29 smithi145 ceph-mon[21899]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-04-22T15:08:29.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:29 smithi145 ceph-mon[26606]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-04-22T15:08:31.191 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:31 smithi172 ceph-mon[22747]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-22T15:08:31.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:31 smithi145 ceph-mon[21899]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-22T15:08:31.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:31 smithi145 ceph-mon[26606]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-22T15:08:33.112 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:32 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:33.112 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:32 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:08:33.112 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:32 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:33.113 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:32 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:08:33.113 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:32 smithi172 ceph-mon[22747]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 47 KiB/s, 0 objects/s recovering 2024-04-22T15:08:33.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:32 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:33.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:32 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:08:33.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:32 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:33.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:32 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:08:33.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:32 smithi145 ceph-mon[21899]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 47 KiB/s, 0 objects/s recovering 2024-04-22T15:08:33.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:32 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:33.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:32 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:08:33.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:32 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:33.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:32 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:08:33.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:32 smithi145 ceph-mon[26606]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 47 KiB/s, 0 objects/s recovering 2024-04-22T15:08:34.725 INFO:teuthology.orchestra.run.smithi172.stdout:Created osd(s) 5 on host 'smithi172' 2024-04-22T15:08:35.314 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:34 smithi172 ceph-mon[22747]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T15:08:35.314 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:34 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:35.314 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:34 smithi172 ceph-mon[22747]: from='osd.5 [v2:172.21.15.172:6808/878495,v1:172.21.15.172:6809/878495]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T15:08:35.314 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:34 smithi172 ceph-mon[22747]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T15:08:35.336 DEBUG:teuthology.orchestra.run.smithi172:osd.5> sudo journalctl -f -n 0 -u ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.5.service 2024-04-22T15:08:35.339 INFO:tasks.cephadm:Deploying osd.6 on smithi172 with /dev/vg_nvme/lv_2... 2024-04-22T15:08:35.339 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-22T15:08:35.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:34 smithi145 ceph-mon[21899]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T15:08:35.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:34 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:35.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:34 smithi145 ceph-mon[21899]: from='osd.5 [v2:172.21.15.172:6808/878495,v1:172.21.15.172:6809/878495]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T15:08:35.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:34 smithi145 ceph-mon[21899]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T15:08:35.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:34 smithi145 ceph-mon[26606]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T15:08:35.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:34 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:35.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:34 smithi145 ceph-mon[26606]: from='osd.5 [v2:172.21.15.172:6808/878495,v1:172.21.15.172:6809/878495]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T15:08:35.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:34 smithi145 ceph-mon[26606]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T15:08:36.744 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:36 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:36.744 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:36 smithi172 ceph-mon[22747]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-22T15:08:36.744 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:36 smithi172 ceph-mon[22747]: osdmap e37: 6 total, 5 up, 6 in 2024-04-22T15:08:36.744 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:36 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:08:36.744 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:36 smithi172 ceph-mon[22747]: from='osd.5 [v2:172.21.15.172:6808/878495,v1:172.21.15.172:6809/878495]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:08:36.745 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:36 smithi172 ceph-mon[22747]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:08:36.745 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:36 smithi172 ceph-mon[22747]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T15:08:36.745 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:08:36 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[29081]: 2024-04-22T15:08:36.733+0000 7fe5951a6700 -1 osd.5 0 waiting for initial osdmap 2024-04-22T15:08:36.745 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:08:36 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[29081]: 2024-04-22T15:08:36.740+0000 7fe58f33c700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T15:08:36.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:36 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:36.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:36 smithi145 ceph-mon[21899]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-22T15:08:36.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:36 smithi145 ceph-mon[21899]: osdmap e37: 6 total, 5 up, 6 in 2024-04-22T15:08:36.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:36 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:08:36.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:36 smithi145 ceph-mon[21899]: from='osd.5 [v2:172.21.15.172:6808/878495,v1:172.21.15.172:6809/878495]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:08:36.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:36 smithi145 ceph-mon[21899]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:08:36.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:36 smithi145 ceph-mon[21899]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T15:08:36.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:36 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:36.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:36 smithi145 ceph-mon[26606]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-22T15:08:36.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:36 smithi145 ceph-mon[26606]: osdmap e37: 6 total, 5 up, 6 in 2024-04-22T15:08:36.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:36 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:08:36.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:36 smithi145 ceph-mon[26606]: from='osd.5 [v2:172.21.15.172:6808/878495,v1:172.21.15.172:6809/878495]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:08:36.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:36 smithi145 ceph-mon[26606]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:08:36.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:36 smithi145 ceph-mon[26606]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T15:08:37.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:37 smithi172 ceph-mon[22747]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]': finished 2024-04-22T15:08:37.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:37 smithi172 ceph-mon[22747]: osdmap e38: 6 total, 5 up, 6 in 2024-04-22T15:08:37.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:37 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:08:37.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:37 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:08:38.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:37 smithi145 ceph-mon[21899]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]': finished 2024-04-22T15:08:38.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:37 smithi145 ceph-mon[21899]: osdmap e38: 6 total, 5 up, 6 in 2024-04-22T15:08:38.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:37 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:08:38.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:37 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:08:38.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:37 smithi145 ceph-mon[26606]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]': finished 2024-04-22T15:08:38.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:37 smithi145 ceph-mon[26606]: osdmap e38: 6 total, 5 up, 6 in 2024-04-22T15:08:38.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:37 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:08:38.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:37 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:08:38.206 INFO:teuthology.orchestra.run.smithi172.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-22T15:08:38.206 INFO:teuthology.orchestra.run.smithi172.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-22T15:08:38.206 INFO:teuthology.orchestra.run.smithi172.stderr: stderr: 10+0 records in 2024-04-22T15:08:38.206 INFO:teuthology.orchestra.run.smithi172.stderr:10+0 records out 2024-04-22T15:08:38.206 INFO:teuthology.orchestra.run.smithi172.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0126479 s, 829 MB/s 2024-04-22T15:08:38.206 INFO:teuthology.orchestra.run.smithi172.stderr:--> Zapping successful for: 2024-04-22T15:08:38.959 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi172:vg_nvme/lv_2 2024-04-22T15:08:39.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:38 smithi145 ceph-mon[21899]: purged_snaps scrub starts 2024-04-22T15:08:39.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:38 smithi145 ceph-mon[21899]: purged_snaps scrub ok 2024-04-22T15:08:39.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:38 smithi145 ceph-mon[21899]: osd.5 [v2:172.21.15.172:6808/878495,v1:172.21.15.172:6809/878495] boot 2024-04-22T15:08:39.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:38 smithi145 ceph-mon[21899]: osdmap e39: 6 total, 6 up, 6 in 2024-04-22T15:08:39.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:38 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:08:39.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:38 smithi145 ceph-mon[21899]: pgmap v128: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:08:39.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:38 smithi145 ceph-mon[21899]: osdmap e40: 6 total, 6 up, 6 in 2024-04-22T15:08:39.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:38 smithi145 ceph-mon[26606]: purged_snaps scrub starts 2024-04-22T15:08:39.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:38 smithi145 ceph-mon[26606]: purged_snaps scrub ok 2024-04-22T15:08:39.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:38 smithi145 ceph-mon[26606]: osd.5 [v2:172.21.15.172:6808/878495,v1:172.21.15.172:6809/878495] boot 2024-04-22T15:08:39.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:38 smithi145 ceph-mon[26606]: osdmap e39: 6 total, 6 up, 6 in 2024-04-22T15:08:39.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:38 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:08:39.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:38 smithi145 ceph-mon[26606]: pgmap v128: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:08:39.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:38 smithi145 ceph-mon[26606]: osdmap e40: 6 total, 6 up, 6 in 2024-04-22T15:08:39.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:38 smithi172 ceph-mon[22747]: purged_snaps scrub starts 2024-04-22T15:08:39.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:38 smithi172 ceph-mon[22747]: purged_snaps scrub ok 2024-04-22T15:08:39.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:38 smithi172 ceph-mon[22747]: osd.5 [v2:172.21.15.172:6808/878495,v1:172.21.15.172:6809/878495] boot 2024-04-22T15:08:39.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:38 smithi172 ceph-mon[22747]: osdmap e39: 6 total, 6 up, 6 in 2024-04-22T15:08:39.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:38 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:08:39.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:38 smithi172 ceph-mon[22747]: pgmap v128: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:08:39.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:38 smithi172 ceph-mon[22747]: osdmap e40: 6 total, 6 up, 6 in 2024-04-22T15:08:39.807 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:39 smithi172 ceph-mon[22747]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-22T15:08:39.808 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:39 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:39.808 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:39 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:08:39.808 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:39 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:08:39.808 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:39 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:39.808 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:39 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:39.808 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:39 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:08:39.808 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:39 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:39.808 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:39 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:08:39.808 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:39 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:39.808 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:39 smithi172 ceph-mon[22747]: osdmap e41: 6 total, 6 up, 6 in 2024-04-22T15:08:40.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:39 smithi145 ceph-mon[21899]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-22T15:08:40.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:39 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:40.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:39 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:08:40.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:39 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:08:40.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:39 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:40.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:39 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:40.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:39 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:08:40.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:39 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:40.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:39 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:08:40.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:39 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:40.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:39 smithi145 ceph-mon[21899]: osdmap e41: 6 total, 6 up, 6 in 2024-04-22T15:08:40.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:39 smithi145 ceph-mon[26606]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-22T15:08:40.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:39 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:40.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:39 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:08:40.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:39 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:08:40.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:39 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:40.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:39 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:40.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:39 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:08:40.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:39 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:40.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:39 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:08:40.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:39 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:40.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:39 smithi145 ceph-mon[26606]: osdmap e41: 6 total, 6 up, 6 in 2024-04-22T15:08:40.938 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:40 smithi172 ceph-mon[22747]: Detected new or changed devices on smithi172 2024-04-22T15:08:40.938 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:40 smithi172 ceph-mon[22747]: Adjusting osd_memory_target on smithi172 to 8544M 2024-04-22T15:08:40.938 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:40 smithi172 ceph-mon[22747]: pgmap v131: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:08:41.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:40 smithi145 ceph-mon[21899]: Detected new or changed devices on smithi172 2024-04-22T15:08:41.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:40 smithi145 ceph-mon[21899]: Adjusting osd_memory_target on smithi172 to 8544M 2024-04-22T15:08:41.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:40 smithi145 ceph-mon[21899]: pgmap v131: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:08:41.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:40 smithi145 ceph-mon[26606]: Detected new or changed devices on smithi172 2024-04-22T15:08:41.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:40 smithi145 ceph-mon[26606]: Adjusting osd_memory_target on smithi172 to 8544M 2024-04-22T15:08:41.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:40 smithi145 ceph-mon[26606]: pgmap v131: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:08:41.841 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:41 smithi172 ceph-mon[22747]: osdmap e42: 6 total, 6 up, 6 in 2024-04-22T15:08:41.841 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:41 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:08:41.841 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:41 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:08:41.841 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:41 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:42.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:41 smithi145 ceph-mon[21899]: osdmap e42: 6 total, 6 up, 6 in 2024-04-22T15:08:42.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:41 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:08:42.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:41 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:08:42.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:41 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:42.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:41 smithi145 ceph-mon[26606]: osdmap e42: 6 total, 6 up, 6 in 2024-04-22T15:08:42.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:41 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:08:42.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:41 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:08:42.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:41 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:42.864 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:42 smithi172 ceph-mon[22747]: from='client.24263 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi172:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:08:42.864 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:42 smithi172 ceph-mon[22747]: osdmap e43: 6 total, 6 up, 6 in 2024-04-22T15:08:42.864 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:42 smithi172 ceph-mon[22747]: pgmap v134: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:08:43.024 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:42 smithi145 ceph-mon[21899]: from='client.24263 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi172:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:08:43.024 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:42 smithi145 ceph-mon[21899]: osdmap e43: 6 total, 6 up, 6 in 2024-04-22T15:08:43.025 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:42 smithi145 ceph-mon[21899]: pgmap v134: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:08:43.025 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:42 smithi145 ceph-mon[26606]: from='client.24263 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi172:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:08:43.025 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:42 smithi145 ceph-mon[26606]: osdmap e43: 6 total, 6 up, 6 in 2024-04-22T15:08:43.025 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:42 smithi145 ceph-mon[26606]: pgmap v134: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:08:44.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:43 smithi145 ceph-mon[21899]: from='client.? 172.21.15.172:0/1511828865' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ff30003d-cc8d-4e7b-beb0-56a4813be817"}]: dispatch 2024-04-22T15:08:44.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:43 smithi145 ceph-mon[21899]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ff30003d-cc8d-4e7b-beb0-56a4813be817"}]: dispatch 2024-04-22T15:08:44.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:43 smithi145 ceph-mon[21899]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ff30003d-cc8d-4e7b-beb0-56a4813be817"}]': finished 2024-04-22T15:08:44.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:43 smithi145 ceph-mon[21899]: osdmap e44: 7 total, 6 up, 7 in 2024-04-22T15:08:44.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:43 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:08:44.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:43 smithi145 ceph-mon[26606]: from='client.? 172.21.15.172:0/1511828865' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ff30003d-cc8d-4e7b-beb0-56a4813be817"}]: dispatch 2024-04-22T15:08:44.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:43 smithi145 ceph-mon[26606]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ff30003d-cc8d-4e7b-beb0-56a4813be817"}]: dispatch 2024-04-22T15:08:44.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:43 smithi145 ceph-mon[26606]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ff30003d-cc8d-4e7b-beb0-56a4813be817"}]': finished 2024-04-22T15:08:44.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:43 smithi145 ceph-mon[26606]: osdmap e44: 7 total, 6 up, 7 in 2024-04-22T15:08:44.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:43 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:08:44.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:43 smithi172 ceph-mon[22747]: from='client.? 172.21.15.172:0/1511828865' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ff30003d-cc8d-4e7b-beb0-56a4813be817"}]: dispatch 2024-04-22T15:08:44.226 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:43 smithi172 ceph-mon[22747]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ff30003d-cc8d-4e7b-beb0-56a4813be817"}]: dispatch 2024-04-22T15:08:44.226 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:43 smithi172 ceph-mon[22747]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ff30003d-cc8d-4e7b-beb0-56a4813be817"}]': finished 2024-04-22T15:08:44.226 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:43 smithi172 ceph-mon[22747]: osdmap e44: 7 total, 6 up, 7 in 2024-04-22T15:08:44.226 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:43 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:08:45.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:44 smithi145 ceph-mon[21899]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 124 KiB/s, 0 objects/s recovering 2024-04-22T15:08:45.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:44 smithi145 ceph-mon[21899]: from='client.? 172.21.15.172:0/2030713825' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:08:45.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:44 smithi145 ceph-mon[21899]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-22T15:08:45.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:44 smithi145 ceph-mon[21899]: Cluster is now healthy 2024-04-22T15:08:45.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:44 smithi145 ceph-mon[26606]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 124 KiB/s, 0 objects/s recovering 2024-04-22T15:08:45.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:44 smithi145 ceph-mon[26606]: from='client.? 172.21.15.172:0/2030713825' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:08:45.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:44 smithi145 ceph-mon[26606]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-22T15:08:45.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:44 smithi145 ceph-mon[26606]: Cluster is now healthy 2024-04-22T15:08:45.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:44 smithi172 ceph-mon[22747]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 124 KiB/s, 0 objects/s recovering 2024-04-22T15:08:45.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:44 smithi172 ceph-mon[22747]: from='client.? 172.21.15.172:0/2030713825' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:08:45.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:44 smithi172 ceph-mon[22747]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-22T15:08:45.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:44 smithi172 ceph-mon[22747]: Cluster is now healthy 2024-04-22T15:08:47.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:46 smithi172 ceph-mon[22747]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-22T15:08:47.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:46 smithi145 ceph-mon[21899]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-22T15:08:47.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:46 smithi145 ceph-mon[26606]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-22T15:08:49.090 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:48 smithi172 ceph-mon[22747]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-22T15:08:49.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:48 smithi145 ceph-mon[21899]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-22T15:08:49.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:49 smithi145 ceph-mon[26606]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-22T15:08:50.210 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:50 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T15:08:50.210 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:50 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:50.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:50 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T15:08:50.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:50 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:50.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:50 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T15:08:50.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:50 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:51.050 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:51 smithi172 ceph-mon[22747]: Deploying daemon osd.6 on smithi172 2024-04-22T15:08:51.050 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:51 smithi172 ceph-mon[22747]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-04-22T15:08:51.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:51 smithi145 ceph-mon[21899]: Deploying daemon osd.6 on smithi172 2024-04-22T15:08:51.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:51 smithi145 ceph-mon[21899]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-04-22T15:08:51.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:51 smithi145 ceph-mon[26606]: Deploying daemon osd.6 on smithi172 2024-04-22T15:08:51.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:51 smithi145 ceph-mon[26606]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-04-22T15:08:53.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:53 smithi172 ceph-mon[22747]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-22T15:08:53.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:53 smithi145 ceph-mon[21899]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-22T15:08:53.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:53 smithi145 ceph-mon[26606]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-22T15:08:55.108 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:55 smithi172 ceph-mon[22747]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:08:55.109 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:55 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:55.109 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:55 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:08:55.109 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:55 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:55.109 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:55 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:08:55.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:55 smithi145 ceph-mon[26606]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:08:55.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:55 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:55.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:55 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:08:55.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:55 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:55.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:55 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:08:55.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:55 smithi145 ceph-mon[21899]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:08:55.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:55 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:55.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:55 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:08:55.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:55 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:08:55.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:55 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:08:57.362 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:57 smithi172 ceph-mon[22747]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:08:57.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:57 smithi145 ceph-mon[21899]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:08:57.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:57 smithi145 ceph-mon[26606]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:08:57.670 INFO:teuthology.orchestra.run.smithi172.stdout:Created osd(s) 6 on host 'smithi172' 2024-04-22T15:08:58.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:58 smithi172 ceph-mon[22747]: from='osd.6 [v2:172.21.15.172:6816/2239493705,v1:172.21.15.172:6817/2239493705]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T15:08:58.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:58 smithi172 ceph-mon[22747]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T15:08:58.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:58 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:58.283 DEBUG:teuthology.orchestra.run.smithi172:osd.6> sudo journalctl -f -n 0 -u ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.6.service 2024-04-22T15:08:58.285 INFO:tasks.cephadm:Deploying osd.7 on smithi172 with /dev/vg_nvme/lv_1... 2024-04-22T15:08:58.286 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-22T15:08:58.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:58 smithi145 ceph-mon[21899]: from='osd.6 [v2:172.21.15.172:6816/2239493705,v1:172.21.15.172:6817/2239493705]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T15:08:58.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:58 smithi145 ceph-mon[21899]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T15:08:58.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:58 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:58.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:58 smithi145 ceph-mon[26606]: from='osd.6 [v2:172.21.15.172:6816/2239493705,v1:172.21.15.172:6817/2239493705]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T15:08:58.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:58 smithi145 ceph-mon[26606]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T15:08:58.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:58 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:59.045 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:08:59 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[31955]: 2024-04-22T15:08:59.034+0000 7fe78baae700 -1 osd.6 0 waiting for initial osdmap 2024-04-22T15:08:59.045 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:08:59 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[31955]: 2024-04-22T15:08:59.041+0000 7fe784441700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T15:08:59.045 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:59 smithi172 ceph-mon[22747]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:08:59.045 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:59 smithi172 ceph-mon[22747]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-22T15:08:59.045 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:59 smithi172 ceph-mon[22747]: osdmap e45: 7 total, 6 up, 7 in 2024-04-22T15:08:59.045 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:59 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:08:59.045 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:59 smithi172 ceph-mon[22747]: from='osd.6 [v2:172.21.15.172:6816/2239493705,v1:172.21.15.172:6817/2239493705]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:08:59.045 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:59 smithi172 ceph-mon[22747]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:08:59.045 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:08:59 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:59.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:59 smithi145 ceph-mon[21899]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:08:59.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:59 smithi145 ceph-mon[21899]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-22T15:08:59.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:59 smithi145 ceph-mon[21899]: osdmap e45: 7 total, 6 up, 7 in 2024-04-22T15:08:59.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:59 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:08:59.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:59 smithi145 ceph-mon[21899]: from='osd.6 [v2:172.21.15.172:6816/2239493705,v1:172.21.15.172:6817/2239493705]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:08:59.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:59 smithi145 ceph-mon[21899]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:08:59.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:08:59 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:08:59.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:59 smithi145 ceph-mon[26606]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:08:59.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:59 smithi145 ceph-mon[26606]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-22T15:08:59.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:59 smithi145 ceph-mon[26606]: osdmap e45: 7 total, 6 up, 7 in 2024-04-22T15:08:59.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:59 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:08:59.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:59 smithi145 ceph-mon[26606]: from='osd.6 [v2:172.21.15.172:6816/2239493705,v1:172.21.15.172:6817/2239493705]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:08:59.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:59 smithi145 ceph-mon[26606]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:08:59.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:08:59 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:00.352 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:00 smithi172 ceph-mon[22747]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]': finished 2024-04-22T15:09:00.352 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:00 smithi172 ceph-mon[22747]: osdmap e46: 7 total, 6 up, 7 in 2024-04-22T15:09:00.352 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:00 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:09:00.353 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:00 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:09:00.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:00 smithi145 ceph-mon[21899]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]': finished 2024-04-22T15:09:00.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:00 smithi145 ceph-mon[21899]: osdmap e46: 7 total, 6 up, 7 in 2024-04-22T15:09:00.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:00 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:09:00.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:00 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:09:00.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:00 smithi145 ceph-mon[26606]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]': finished 2024-04-22T15:09:00.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:00 smithi145 ceph-mon[26606]: osdmap e46: 7 total, 6 up, 7 in 2024-04-22T15:09:00.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:00 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:09:00.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:00 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:09:00.936 INFO:teuthology.orchestra.run.smithi172.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-22T15:09:00.936 INFO:teuthology.orchestra.run.smithi172.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-22T15:09:00.936 INFO:teuthology.orchestra.run.smithi172.stderr: stderr: 10+0 records in 2024-04-22T15:09:00.936 INFO:teuthology.orchestra.run.smithi172.stderr:10+0 records out 2024-04-22T15:09:00.936 INFO:teuthology.orchestra.run.smithi172.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0152064 s, 690 MB/s 2024-04-22T15:09:00.936 INFO:teuthology.orchestra.run.smithi172.stderr:--> Zapping successful for: 2024-04-22T15:09:01.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:01 smithi172 ceph-mon[22747]: purged_snaps scrub starts 2024-04-22T15:09:01.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:01 smithi172 ceph-mon[22747]: purged_snaps scrub ok 2024-04-22T15:09:01.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:01 smithi172 ceph-mon[22747]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:09:01.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:01 smithi172 ceph-mon[22747]: osd.6 [v2:172.21.15.172:6816/2239493705,v1:172.21.15.172:6817/2239493705] boot 2024-04-22T15:09:01.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:01 smithi172 ceph-mon[22747]: osdmap e47: 7 total, 7 up, 7 in 2024-04-22T15:09:01.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:01 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:09:01.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:01 smithi172 ceph-mon[22747]: osdmap e48: 7 total, 7 up, 7 in 2024-04-22T15:09:01.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:01 smithi145 ceph-mon[21899]: purged_snaps scrub starts 2024-04-22T15:09:01.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:01 smithi145 ceph-mon[21899]: purged_snaps scrub ok 2024-04-22T15:09:01.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:01 smithi145 ceph-mon[21899]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:09:01.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:01 smithi145 ceph-mon[21899]: osd.6 [v2:172.21.15.172:6816/2239493705,v1:172.21.15.172:6817/2239493705] boot 2024-04-22T15:09:01.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:01 smithi145 ceph-mon[21899]: osdmap e47: 7 total, 7 up, 7 in 2024-04-22T15:09:01.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:01 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:09:01.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:01 smithi145 ceph-mon[21899]: osdmap e48: 7 total, 7 up, 7 in 2024-04-22T15:09:01.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:01 smithi145 ceph-mon[26606]: purged_snaps scrub starts 2024-04-22T15:09:01.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:01 smithi145 ceph-mon[26606]: purged_snaps scrub ok 2024-04-22T15:09:01.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:01 smithi145 ceph-mon[26606]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T15:09:01.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:01 smithi145 ceph-mon[26606]: osd.6 [v2:172.21.15.172:6816/2239493705,v1:172.21.15.172:6817/2239493705] boot 2024-04-22T15:09:01.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:01 smithi145 ceph-mon[26606]: osdmap e47: 7 total, 7 up, 7 in 2024-04-22T15:09:01.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:01 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:09:01.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:01 smithi145 ceph-mon[26606]: osdmap e48: 7 total, 7 up, 7 in 2024-04-22T15:09:01.625 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi172:vg_nvme/lv_1 2024-04-22T15:09:03.280 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:03 smithi172 ceph-mon[22747]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-22T15:09:03.280 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:03 smithi172 ceph-mon[22747]: osdmap e49: 7 total, 7 up, 7 in 2024-04-22T15:09:03.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:03 smithi145 ceph-mon[21899]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-22T15:09:03.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:03 smithi145 ceph-mon[21899]: osdmap e49: 7 total, 7 up, 7 in 2024-04-22T15:09:03.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:03 smithi145 ceph-mon[26606]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-22T15:09:03.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:03 smithi145 ceph-mon[26606]: osdmap e49: 7 total, 7 up, 7 in 2024-04-22T15:09:04.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:04 smithi172 ceph-mon[22747]: osdmap e50: 7 total, 7 up, 7 in 2024-04-22T15:09:04.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:04 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:04.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:04 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:04.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:04 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:04.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:04 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:04.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:04 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:04.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:04 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:04.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:04 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:09:04.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:04 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:04.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:04 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:09:04.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:04 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:04.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:04 smithi145 ceph-mon[26606]: osdmap e50: 7 total, 7 up, 7 in 2024-04-22T15:09:04.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:04 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:04.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:04 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:04.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:04 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:04.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:04 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:04.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:04 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:04.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:04 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:04.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:04 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:09:04.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:04 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:04.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:04 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:09:04.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:04 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:04.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:04 smithi145 ceph-mon[21899]: osdmap e50: 7 total, 7 up, 7 in 2024-04-22T15:09:04.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:04 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:04.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:04 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:04.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:04 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:04.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:04 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:04.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:04 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:04.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:04 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:04.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:04 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:09:04.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:04 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:04.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:04 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:09:04.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:04 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:05.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:05 smithi145 ceph-mon[21899]: Detected new or changed devices on smithi172 2024-04-22T15:09:05.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:05 smithi145 ceph-mon[21899]: Adjusting osd_memory_target on smithi172 to 5696M 2024-04-22T15:09:05.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:05 smithi145 ceph-mon[21899]: pgmap v152: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-04-22T15:09:05.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:05 smithi145 ceph-mon[21899]: osdmap e51: 7 total, 7 up, 7 in 2024-04-22T15:09:05.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:05 smithi145 ceph-mon[21899]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-04-22T15:09:05.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:05 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:09:05.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:05 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:09:05.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:05 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:05.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:05 smithi145 ceph-mon[26606]: Detected new or changed devices on smithi172 2024-04-22T15:09:05.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:05 smithi145 ceph-mon[26606]: Adjusting osd_memory_target on smithi172 to 5696M 2024-04-22T15:09:05.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:05 smithi145 ceph-mon[26606]: pgmap v152: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-04-22T15:09:05.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:05 smithi145 ceph-mon[26606]: osdmap e51: 7 total, 7 up, 7 in 2024-04-22T15:09:05.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:05 smithi145 ceph-mon[26606]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-04-22T15:09:05.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:05 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:09:05.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:05 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:09:05.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:05 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:05.416 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:05 smithi172 ceph-mon[22747]: Detected new or changed devices on smithi172 2024-04-22T15:09:05.416 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:05 smithi172 ceph-mon[22747]: Adjusting osd_memory_target on smithi172 to 5696M 2024-04-22T15:09:05.416 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:05 smithi172 ceph-mon[22747]: pgmap v152: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-04-22T15:09:05.416 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:05 smithi172 ceph-mon[22747]: osdmap e51: 7 total, 7 up, 7 in 2024-04-22T15:09:05.416 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:05 smithi172 ceph-mon[22747]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-04-22T15:09:05.417 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:05 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T15:09:05.417 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:05 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T15:09:05.417 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:05 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:06.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:06 smithi172 ceph-mon[22747]: from='client.24290 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi172:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:09:06.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:06 smithi145 ceph-mon[21899]: from='client.24290 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi172:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:09:06.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:06 smithi145 ceph-mon[26606]: from='client.24290 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi172:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:09:07.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:07 smithi145 ceph-mon[21899]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 106 KiB/s, 0 objects/s recovering 2024-04-22T15:09:07.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:07 smithi145 ceph-mon[21899]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-04-22T15:09:07.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:07 smithi145 ceph-mon[21899]: Cluster is now healthy 2024-04-22T15:09:07.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:07 smithi145 ceph-mon[21899]: from='client.? 172.21.15.172:0/296201793' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7723e772-46df-4001-8f36-83b3695c9465"}]: dispatch 2024-04-22T15:09:07.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:07 smithi145 ceph-mon[21899]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7723e772-46df-4001-8f36-83b3695c9465"}]: dispatch 2024-04-22T15:09:07.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:07 smithi145 ceph-mon[21899]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7723e772-46df-4001-8f36-83b3695c9465"}]': finished 2024-04-22T15:09:07.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:07 smithi145 ceph-mon[21899]: osdmap e52: 8 total, 7 up, 8 in 2024-04-22T15:09:07.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:07 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:09:07.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:07 smithi145 ceph-mon[26606]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 106 KiB/s, 0 objects/s recovering 2024-04-22T15:09:07.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:07 smithi145 ceph-mon[26606]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-04-22T15:09:07.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:07 smithi145 ceph-mon[26606]: Cluster is now healthy 2024-04-22T15:09:07.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:07 smithi145 ceph-mon[26606]: from='client.? 172.21.15.172:0/296201793' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7723e772-46df-4001-8f36-83b3695c9465"}]: dispatch 2024-04-22T15:09:07.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:07 smithi145 ceph-mon[26606]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7723e772-46df-4001-8f36-83b3695c9465"}]: dispatch 2024-04-22T15:09:07.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:07 smithi145 ceph-mon[26606]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7723e772-46df-4001-8f36-83b3695c9465"}]': finished 2024-04-22T15:09:07.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:07 smithi145 ceph-mon[26606]: osdmap e52: 8 total, 7 up, 8 in 2024-04-22T15:09:07.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:07 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:09:07.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:07 smithi172 ceph-mon[22747]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 106 KiB/s, 0 objects/s recovering 2024-04-22T15:09:07.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:07 smithi172 ceph-mon[22747]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-04-22T15:09:07.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:07 smithi172 ceph-mon[22747]: Cluster is now healthy 2024-04-22T15:09:07.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:07 smithi172 ceph-mon[22747]: from='client.? 172.21.15.172:0/296201793' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7723e772-46df-4001-8f36-83b3695c9465"}]: dispatch 2024-04-22T15:09:07.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:07 smithi172 ceph-mon[22747]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7723e772-46df-4001-8f36-83b3695c9465"}]: dispatch 2024-04-22T15:09:07.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:07 smithi172 ceph-mon[22747]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7723e772-46df-4001-8f36-83b3695c9465"}]': finished 2024-04-22T15:09:07.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:07 smithi172 ceph-mon[22747]: osdmap e52: 8 total, 7 up, 8 in 2024-04-22T15:09:07.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:07 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:09:08.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:08 smithi145 ceph-mon[21899]: from='client.? 172.21.15.172:0/2115970463' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:09:08.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:08 smithi145 ceph-mon[26606]: from='client.? 172.21.15.172:0/2115970463' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:09:08.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:08 smithi172 ceph-mon[22747]: from='client.? 172.21.15.172:0/2115970463' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T15:09:09.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:09 smithi145 ceph-mon[21899]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 88 KiB/s, 0 objects/s recovering 2024-04-22T15:09:09.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:09 smithi145 ceph-mon[26606]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 88 KiB/s, 0 objects/s recovering 2024-04-22T15:09:09.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:09 smithi172 ceph-mon[22747]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 88 KiB/s, 0 objects/s recovering 2024-04-22T15:09:11.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:11 smithi145 ceph-mon[21899]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-04-22T15:09:11.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:11 smithi145 ceph-mon[26606]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-04-22T15:09:11.405 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:11 smithi172 ceph-mon[22747]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-04-22T15:09:13.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:13 smithi145 ceph-mon[21899]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-22T15:09:13.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:13 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T15:09:13.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:13 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:13.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:13 smithi145 ceph-mon[26606]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-22T15:09:13.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:13 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T15:09:13.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:13 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:13.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:13 smithi172 ceph-mon[22747]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-22T15:09:13.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:13 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T15:09:13.375 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:13 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:14.341 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:14 smithi172 ceph-mon[22747]: Deploying daemon osd.7 on smithi172 2024-04-22T15:09:14.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:14 smithi145 ceph-mon[26606]: Deploying daemon osd.7 on smithi172 2024-04-22T15:09:14.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:14 smithi145 ceph-mon[21899]: Deploying daemon osd.7 on smithi172 2024-04-22T15:09:15.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:15 smithi145 ceph-mon[21899]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-22T15:09:15.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:15 smithi145 ceph-mon[26606]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-22T15:09:15.448 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:15 smithi172 ceph-mon[22747]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-22T15:09:17.199 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:17 smithi172 ceph-mon[22747]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T15:09:17.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:17 smithi145 ceph-mon[21899]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T15:09:17.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:17 smithi145 ceph-mon[26606]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T15:09:19.034 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:18 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:19.034 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:18 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:09:19.034 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:18 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:19.034 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:18 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:09:19.034 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:18 smithi172 ceph-mon[22747]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T15:09:19.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:18 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:19.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:18 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:09:19.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:18 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:19.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:18 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:09:19.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:18 smithi145 ceph-mon[21899]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T15:09:19.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:18 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:19.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:18 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:09:19.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:18 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:19.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:18 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:09:19.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:18 smithi145 ceph-mon[26606]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T15:09:20.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:20 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:09:20.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:20 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:09:20.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:20 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:09:20.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:20 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:09:20.410 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:20 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:09:20.410 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:20 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:09:20.775 INFO:teuthology.orchestra.run.smithi172.stdout:Created osd(s) 7 on host 'smithi172' 2024-04-22T15:09:21.355 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:21 smithi172 ceph-mon[22747]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T15:09:21.356 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:21 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:21.356 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:21 smithi172 ceph-mon[22747]: from='osd.7 [v2:172.21.15.172:6824/2431704909,v1:172.21.15.172:6825/2431704909]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T15:09:21.356 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:21 smithi172 ceph-mon[22747]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T15:09:21.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:21 smithi145 ceph-mon[21899]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T15:09:21.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:21 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:21.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:21 smithi145 ceph-mon[21899]: from='osd.7 [v2:172.21.15.172:6824/2431704909,v1:172.21.15.172:6825/2431704909]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T15:09:21.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:21 smithi145 ceph-mon[21899]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T15:09:21.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:21 smithi145 ceph-mon[26606]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T15:09:21.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:21 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:21.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:21 smithi145 ceph-mon[26606]: from='osd.7 [v2:172.21.15.172:6824/2431704909,v1:172.21.15.172:6825/2431704909]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T15:09:21.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:21 smithi145 ceph-mon[26606]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T15:09:21.378 DEBUG:teuthology.orchestra.run.smithi172:osd.7> sudo journalctl -f -n 0 -u ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.7.service 2024-04-22T15:09:21.380 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-04-22T15:09:21.381 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph osd stat -f json 2024-04-22T15:09:22.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:22 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:22.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:22 smithi172 ceph-mon[22747]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-22T15:09:22.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:22 smithi172 ceph-mon[22747]: osdmap e53: 8 total, 7 up, 8 in 2024-04-22T15:09:22.870 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:22 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:09:22.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:22 smithi172 ceph-mon[22747]: from='osd.7 [v2:172.21.15.172:6824/2431704909,v1:172.21.15.172:6825/2431704909]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:09:22.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:22 smithi172 ceph-mon[22747]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:09:22.871 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:22 smithi172 ceph-mon[22747]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T15:09:22.871 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:09:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[34946]: 2024-04-22T15:09:22.792+0000 7f063ba06700 -1 osd.7 0 waiting for initial osdmap 2024-04-22T15:09:22.871 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:09:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[34946]: 2024-04-22T15:09:22.802+0000 7f063539b700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T15:09:22.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:22 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:22.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:22 smithi145 ceph-mon[21899]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-22T15:09:22.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:22 smithi145 ceph-mon[21899]: osdmap e53: 8 total, 7 up, 8 in 2024-04-22T15:09:22.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:22 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:09:22.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:22 smithi145 ceph-mon[21899]: from='osd.7 [v2:172.21.15.172:6824/2431704909,v1:172.21.15.172:6825/2431704909]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:09:22.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:22 smithi145 ceph-mon[21899]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:09:22.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:22 smithi145 ceph-mon[21899]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T15:09:22.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:22 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:22.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:22 smithi145 ceph-mon[26606]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-22T15:09:22.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:22 smithi145 ceph-mon[26606]: osdmap e53: 8 total, 7 up, 8 in 2024-04-22T15:09:22.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:22 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:09:22.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:22 smithi145 ceph-mon[26606]: from='osd.7 [v2:172.21.15.172:6824/2431704909,v1:172.21.15.172:6825/2431704909]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:09:22.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:22 smithi145 ceph-mon[26606]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:09:22.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:22 smithi145 ceph-mon[26606]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T15:09:23.514 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-04-22T15:09:24.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:23 smithi145 ceph-mon[21899]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]': finished 2024-04-22T15:09:24.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:23 smithi145 ceph-mon[21899]: osdmap e54: 8 total, 7 up, 8 in 2024-04-22T15:09:24.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:23 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:09:24.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:23 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:09:24.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:23 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3756126764' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-22T15:09:24.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:23 smithi145 ceph-mon[26606]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]': finished 2024-04-22T15:09:24.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:23 smithi145 ceph-mon[26606]: osdmap e54: 8 total, 7 up, 8 in 2024-04-22T15:09:24.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:23 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:09:24.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:23 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:09:24.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:23 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3756126764' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-22T15:09:24.201 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:23 smithi172 ceph-mon[22747]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]': finished 2024-04-22T15:09:24.201 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:23 smithi172 ceph-mon[22747]: osdmap e54: 8 total, 7 up, 8 in 2024-04-22T15:09:24.201 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:23 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:09:24.201 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:23 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:09:24.201 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:23 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/3756126764' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-22T15:09:24.351 INFO:teuthology.orchestra.run.smithi145.stdout:{"epoch":54,"num_osds":8,"num_up_osds":7,"osd_up_since":1713798540,"num_in_osds":8,"osd_in_since":1713798546,"num_remapped_pgs":0} 2024-04-22T15:09:25.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[21899]: purged_snaps scrub starts 2024-04-22T15:09:25.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[21899]: purged_snaps scrub ok 2024-04-22T15:09:25.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[21899]: osd.7 [v2:172.21.15.172:6824/2431704909,v1:172.21.15.172:6825/2431704909] boot 2024-04-22T15:09:25.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[21899]: osdmap e55: 8 total, 8 up, 8 in 2024-04-22T15:09:25.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:09:25.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[21899]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:25.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[21899]: osdmap e56: 8 total, 8 up, 8 in 2024-04-22T15:09:25.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:25.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:25.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:25.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:25.125 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:25.125 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:25.125 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:25.125 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:09:25.125 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:25.125 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:09:25.125 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:25.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[26606]: purged_snaps scrub starts 2024-04-22T15:09:25.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[26606]: purged_snaps scrub ok 2024-04-22T15:09:25.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[26606]: osd.7 [v2:172.21.15.172:6824/2431704909,v1:172.21.15.172:6825/2431704909] boot 2024-04-22T15:09:25.126 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[26606]: osdmap e55: 8 total, 8 up, 8 in 2024-04-22T15:09:25.126 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:09:25.126 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[26606]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:25.126 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[26606]: osdmap e56: 8 total, 8 up, 8 in 2024-04-22T15:09:25.126 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:25.126 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:25.126 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:25.126 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:25.126 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:25.126 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:25.126 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:25.127 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:09:25.127 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:25.127 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:09:25.127 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:24 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:25.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:24 smithi172 ceph-mon[22747]: purged_snaps scrub starts 2024-04-22T15:09:25.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:24 smithi172 ceph-mon[22747]: purged_snaps scrub ok 2024-04-22T15:09:25.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:24 smithi172 ceph-mon[22747]: osd.7 [v2:172.21.15.172:6824/2431704909,v1:172.21.15.172:6825/2431704909] boot 2024-04-22T15:09:25.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:24 smithi172 ceph-mon[22747]: osdmap e55: 8 total, 8 up, 8 in 2024-04-22T15:09:25.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:24 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:09:25.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:24 smithi172 ceph-mon[22747]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:25.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:24 smithi172 ceph-mon[22747]: osdmap e56: 8 total, 8 up, 8 in 2024-04-22T15:09:25.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:24 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:25.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:24 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:25.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:24 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:25.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:24 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:25.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:24 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:09:25.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:24 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:25.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:24 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:25.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:24 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:09:25.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:24 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:25.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:24 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:09:25.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:24 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:25.352 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph osd stat -f json 2024-04-22T15:09:26.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:26 smithi145 ceph-mon[26606]: Detected new or changed devices on smithi172 2024-04-22T15:09:26.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:26 smithi145 ceph-mon[26606]: Adjusting osd_memory_target on smithi172 to 4272M 2024-04-22T15:09:26.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:26 smithi145 ceph-mon[26606]: osdmap e57: 8 total, 8 up, 8 in 2024-04-22T15:09:26.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:26 smithi145 ceph-mon[21899]: Detected new or changed devices on smithi172 2024-04-22T15:09:26.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:26 smithi145 ceph-mon[21899]: Adjusting osd_memory_target on smithi172 to 4272M 2024-04-22T15:09:26.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:26 smithi145 ceph-mon[21899]: osdmap e57: 8 total, 8 up, 8 in 2024-04-22T15:09:26.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:26 smithi172 ceph-mon[22747]: Detected new or changed devices on smithi172 2024-04-22T15:09:26.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:26 smithi172 ceph-mon[22747]: Adjusting osd_memory_target on smithi172 to 4272M 2024-04-22T15:09:26.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:26 smithi172 ceph-mon[22747]: osdmap e57: 8 total, 8 up, 8 in 2024-04-22T15:09:27.296 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:27 smithi145 ceph-mon[21899]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:27.296 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:27 smithi145 ceph-mon[21899]: osdmap e58: 8 total, 8 up, 8 in 2024-04-22T15:09:27.296 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:27 smithi145 ceph-mon[26606]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:27.296 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:27 smithi145 ceph-mon[26606]: osdmap e58: 8 total, 8 up, 8 in 2024-04-22T15:09:27.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:27 smithi172 ceph-mon[22747]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:27.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:27 smithi172 ceph-mon[22747]: osdmap e58: 8 total, 8 up, 8 in 2024-04-22T15:09:27.579 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-04-22T15:09:28.325 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:28 smithi145 ceph-mon[26606]: osdmap e59: 8 total, 8 up, 8 in 2024-04-22T15:09:28.325 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:28 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3683381540' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-22T15:09:28.325 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:28 smithi145 ceph-mon[21899]: osdmap e59: 8 total, 8 up, 8 in 2024-04-22T15:09:28.325 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:28 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3683381540' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-22T15:09:28.348 INFO:teuthology.orchestra.run.smithi145.stdout:{"epoch":59,"num_osds":8,"num_up_osds":8,"osd_up_since":1713798563,"num_in_osds":8,"osd_in_since":1713798546,"num_remapped_pgs":0} 2024-04-22T15:09:28.348 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-22T15:09:28.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:28 smithi172 ceph-mon[22747]: osdmap e59: 8 total, 8 up, 8 in 2024-04-22T15:09:28.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:28 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/3683381540' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-22T15:09:28.557 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T15:09:29.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:29 smithi172 ceph-mon[22747]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:29.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:29 smithi145 ceph-mon[21899]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:29.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:29 smithi145 ceph-mon[26606]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:30.397 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-04-22T15:09:30.397 INFO:teuthology.orchestra.run.smithi145.stdout:{"epoch":59,"fsid":"869afcfa-00b9-11ef-bc93-c7b262605968","created":"2024-04-22T15:04:33.942175+0000","modified":"2024-04-22T15:09:27.003976+0000","last_up_change":"2024-04-22T15:09:23.783719+0000","last_in_change":"2024-04-22T15:09:06.672949+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-22T15:07:30.182778+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"8f61de15-c199-4966-8d5d-8acd8ca2d6c8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6802","nonce":2466746205},{"type":"v1","addr":"172.21.15.145:6803","nonce":2466746205}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6804","nonce":2466746205},{"type":"v1","addr":"172.21.15.145:6805","nonce":2466746205}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6808","nonce":2466746205},{"type":"v1","addr":"172.21.15.145:6809","nonce":2466746205}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6806","nonce":2466746205},{"type":"v1","addr":"172.21.15.145:6807","nonce":2466746205}]},"public_addr":"172.21.15.145:6803/2466746205","cluster_addr":"172.21.15.145:6805/2466746205","heartbeat_back_addr":"172.21.15.145:6809/2466746205","heartbeat_front_addr":"172.21.15.145:6807/2466746205","state":["exists","up"]},{"osd":1,"uuid":"f863b9ae-7cb4-4b74-a2a1-a2384e809318","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6810","nonce":1586792147},{"type":"v1","addr":"172.21.15.145:6811","nonce":1586792147}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6812","nonce":1586792147},{"type":"v1","addr":"172.21.15.145:6813","nonce":1586792147}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6816","nonce":1586792147},{"type":"v1","addr":"172.21.15.145:6817","nonce":1586792147}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6814","nonce":1586792147},{"type":"v1","addr":"172.21.15.145:6815","nonce":1586792147}]},"public_addr":"172.21.15.145:6811/1586792147","cluster_addr":"172.21.15.145:6813/1586792147","heartbeat_back_addr":"172.21.15.145:6817/1586792147","heartbeat_front_addr":"172.21.15.145:6815/1586792147","state":["exists","up"]},{"osd":2,"uuid":"359c8817-27db-4026-891b-909210f255bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6818","nonce":3315449646},{"type":"v1","addr":"172.21.15.145:6819","nonce":3315449646}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6820","nonce":3315449646},{"type":"v1","addr":"172.21.15.145:6821","nonce":3315449646}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6824","nonce":3315449646},{"type":"v1","addr":"172.21.15.145:6825","nonce":3315449646}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6822","nonce":3315449646},{"type":"v1","addr":"172.21.15.145:6823","nonce":3315449646}]},"public_addr":"172.21.15.145:6819/3315449646","cluster_addr":"172.21.15.145:6821/3315449646","heartbeat_back_addr":"172.21.15.145:6825/3315449646","heartbeat_front_addr":"172.21.15.145:6823/3315449646","state":["exists","up"]},{"osd":3,"uuid":"25f5ce78-1153-489a-bdc9-7cbd4ff4cfe5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6826","nonce":922673347},{"type":"v1","addr":"172.21.15.145:6827","nonce":922673347}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6828","nonce":922673347},{"type":"v1","addr":"172.21.15.145:6829","nonce":922673347}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6832","nonce":922673347},{"type":"v1","addr":"172.21.15.145:6833","nonce":922673347}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6830","nonce":922673347},{"type":"v1","addr":"172.21.15.145:6831","nonce":922673347}]},"public_addr":"172.21.15.145:6827/922673347","cluster_addr":"172.21.15.145:6829/922673347","heartbeat_back_addr":"172.21.15.145:6833/922673347","heartbeat_front_addr":"172.21.15.145:6831/922673347","state":["exists","up"]},{"osd":4,"uuid":"eefb46ad-39a3-40bf-a966-65f7a32841f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6800","nonce":3136053699},{"type":"v1","addr":"172.21.15.172:6801","nonce":3136053699}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6802","nonce":3136053699},{"type":"v1","addr":"172.21.15.172:6803","nonce":3136053699}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6806","nonce":3136053699},{"type":"v1","addr":"172.21.15.172:6807","nonce":3136053699}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6804","nonce":3136053699},{"type":"v1","addr":"172.21.15.172:6805","nonce":3136053699}]},"public_addr":"172.21.15.172:6801/3136053699","cluster_addr":"172.21.15.172:6803/3136053699","heartbeat_back_addr":"172.21.15.172:6807/3136053699","heartbeat_front_addr":"172.21.15.172:6805/3136053699","state":["exists","up"]},{"osd":5,"uuid":"58c9f92d-ff6c-4774-a08a-e2377ea0e95f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6808","nonce":878495},{"type":"v1","addr":"172.21.15.172:6809","nonce":878495}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6810","nonce":878495},{"type":"v1","addr":"172.21.15.172:6811","nonce":878495}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6814","nonce":878495},{"type":"v1","addr":"172.21.15.172:6815","nonce":878495}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6812","nonce":878495},{"type":"v1","addr":"172.21.15.172:6813","nonce":878495}]},"public_addr":"172.21.15.172:6809/878495","cluster_addr":"172.21.15.172:6811/878495","heartbeat_back_addr":"172.21.15.172:6815/878495","heartbeat_front_addr":"172.21.15.172:6813/878495","state":["exists","up"]},{"osd":6,"uuid":"ff30003d-cc8d-4e7b-beb0-56a4813be817","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6816","nonce":2239493705},{"type":"v1","addr":"172.21.15.172:6817","nonce":2239493705}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6818","nonce":2239493705},{"type":"v1","addr":"172.21.15.172:6819","nonce":2239493705}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6822","nonce":2239493705},{"type":"v1","addr":"172.21.15.172:6823","nonce":2239493705}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6820","nonce":2239493705},{"type":"v1","addr":"172.21.15.172:6821","nonce":2239493705}]},"public_addr":"172.21.15.172:6817/2239493705","cluster_addr":"172.21.15.172:6819/2239493705","heartbeat_back_addr":"172.21.15.172:6823/2239493705","heartbeat_front_addr":"172.21.15.172:6821/2239493705","state":["exists","up"]},{"osd":7,"uuid":"7723e772-46df-4001-8f36-83b3695c9465","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6824","nonce":2431704909},{"type":"v1","addr":"172.21.15.172:6825","nonce":2431704909}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6826","nonce":2431704909},{"type":"v1","addr":"172.21.15.172:6827","nonce":2431704909}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6830","nonce":2431704909},{"type":"v1","addr":"172.21.15.172:6831","nonce":2431704909}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6828","nonce":2431704909},{"type":"v1","addr":"172.21.15.172:6829","nonce":2431704909}]},"public_addr":"172.21.15.172:6825/2431704909","cluster_addr":"172.21.15.172:6827/2431704909","heartbeat_back_addr":"172.21.15.172:6831/2431704909","heartbeat_front_addr":"172.21.15.172:6829/2431704909","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:06:39.225018+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:07:03.246108+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:07:26.590807+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:07:51.641916+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:08:13.263422+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:08:35.886011+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:08:58.567855+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:09:21.846973+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.145:0/2226900857":"2024-04-23T15:05:19.220973+0000","172.21.15.145:6801/1271607645":"2024-04-23T15:05:19.220973+0000","172.21.15.145:0/1075985196":"2024-04-23T15:04:56.813632+0000","172.21.15.145:0/3793806840":"2024-04-23T15:04:56.813632+0000","172.21.15.145:0/1261892263":"2024-04-23T15:04:56.813632+0000","172.21.15.145:0/1333376048":"2024-04-23T15:05:19.220973+0000","172.21.15.145:6800/4174602953":"2024-04-23T15:04:56.813632+0000","172.21.15.145:0/218026216":"2024-04-23T15:05:19.220973+0000","172.21.15.145:6800/1271607645":"2024-04-23T15:05:19.220973+0000","172.21.15.145:6801/4174602953":"2024-04-23T15:04:56.813632+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-22T15:09:31.038 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-22T15:07:30.182778+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-04-22T15:09:31.038 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-04-22T15:09:31.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:31 smithi172 ceph-mon[22747]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-04-22T15:09:31.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:31 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/277905681' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T15:09:31.244 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T15:09:31.366 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:31 smithi145 ceph-mon[21899]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-04-22T15:09:31.367 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:31 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/277905681' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T15:09:31.367 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:31 smithi145 ceph-mon[26606]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-04-22T15:09:31.367 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:31 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/277905681' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T15:09:33.278 INFO:teuthology.orchestra.run.smithi145.stdout:pg_num: 1 2024-04-22T15:09:33.279 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:33 smithi145 ceph-mon[26606]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-04-22T15:09:33.279 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:33 smithi145 ceph-mon[21899]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-04-22T15:09:33.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:33 smithi172 ceph-mon[22747]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-04-22T15:09:33.942 INFO:tasks.cephadm:Adding prometheus.a on smithi172 2024-04-22T15:09:33.943 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph orch apply prometheus '1;smithi172=a' 2024-04-22T15:09:34.261 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:34 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2857107195' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-22T15:09:34.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:34 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2857107195' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-22T15:09:34.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:34 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2857107195' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-22T15:09:35.293 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:35 smithi172 ceph-mon[22747]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-22T15:09:35.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:35 smithi145 ceph-mon[21899]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-22T15:09:35.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:35 smithi145 ceph-mon[26606]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-22T15:09:35.948 INFO:teuthology.orchestra.run.smithi172.stdout:Scheduled prometheus update... 2024-04-22T15:09:36.506 DEBUG:teuthology.orchestra.run.smithi172:prometheus.a> sudo journalctl -f -n 0 -u ceph-869afcfa-00b9-11ef-bc93-c7b262605968@prometheus.a.service 2024-04-22T15:09:36.509 INFO:tasks.cephadm:Adding node-exporter.a on smithi145 2024-04-22T15:09:36.509 INFO:tasks.cephadm:Adding node-exporter.b on smithi172 2024-04-22T15:09:36.509 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph orch apply node-exporter '2;smithi145=a;smithi172=b' 2024-04-22T15:09:37.135 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:37 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ignoring --setuser ceph since I am not root 2024-04-22T15:09:37.135 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:37 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ignoring --setgroup ceph since I am not root 2024-04-22T15:09:37.136 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:37 smithi172 ceph-mgr[23935]: -- 172.21.15.172:0/1534484409 <== mon.2 v2:172.21.15.172:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55cdd7986340 con 0x55cdd795e800 2024-04-22T15:09:37.136 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:37 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:09:37.131+0000 7f7c615cb000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T15:09:37.136 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:36 smithi172 ceph-mon[22747]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-22T15:09:37.136 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:36 smithi172 ceph-mon[22747]: from='client.24329 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi172=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:09:37.136 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:36 smithi172 ceph-mon[22747]: Saving service prometheus spec with placement smithi172=a;count:1 2024-04-22T15:09:37.136 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:36 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:37.136 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:36 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:09:37.136 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:36 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:37.137 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:36 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:09:37.137 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:36 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:37.137 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:36 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-22T15:09:37.264 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:36 smithi145 ceph-mon[21899]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-22T15:09:37.264 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:36 smithi145 ceph-mon[21899]: from='client.24329 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi172=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:09:37.265 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:36 smithi145 ceph-mon[21899]: Saving service prometheus spec with placement smithi172=a;count:1 2024-04-22T15:09:37.265 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:36 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:37.265 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:36 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:09:37.265 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:36 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:37.265 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:36 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:09:37.265 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:36 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:37.265 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:36 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-22T15:09:37.265 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:36 smithi145 ceph-mon[26606]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-22T15:09:37.266 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:36 smithi145 ceph-mon[26606]: from='client.24329 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi172=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:09:37.266 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:36 smithi145 ceph-mon[26606]: Saving service prometheus spec with placement smithi172=a;count:1 2024-04-22T15:09:37.266 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:36 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:37.266 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:36 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:09:37.266 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:36 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:37.266 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:36 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:09:37.266 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:36 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' 2024-04-22T15:09:37.266 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:36 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-22T15:09:37.266 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ignoring --setuser ceph since I am not root 2024-04-22T15:09:37.267 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ignoring --setgroup ceph since I am not root 2024-04-22T15:09:37.267 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:37 smithi145 ceph-mgr[22135]: -- 172.21.15.145:0/876362905 <== mon.1 v2:172.21.15.145:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55bd80fa4340 con 0x55bd80f7d000 2024-04-22T15:09:37.267 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:09:37.134+0000 7f0414de4000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T15:09:37.267 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:09:37.260+0000 7f0414de4000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T15:09:37.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:37 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:09:37.261+0000 7f7c615cb000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T15:09:37.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:37 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:09:37.360+0000 7f7c615cb000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T15:09:37.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:09:37.355+0000 7f0414de4000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T15:09:37.932 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:37 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:09:37.671+0000 7f7c615cb000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T15:09:37.932 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:37 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:09:37.755+0000 7f7c615cb000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T15:09:37.973 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:09:37.655+0000 7f0414de4000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T15:09:37.973 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:09:37.735+0000 7f0414de4000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T15:09:38.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:37 smithi172 ceph-mon[22747]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-22T15:09:38.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:37 smithi172 ceph-mon[22747]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-22T15:09:38.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:37 smithi145 ceph-mon[21899]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-22T15:09:38.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:37 smithi145 ceph-mon[21899]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-22T15:09:38.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:37 smithi145 ceph-mon[26606]: from='mgr.14150 172.21.15.145:0/2496307069' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-22T15:09:38.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:37 smithi145 ceph-mon[26606]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-22T15:09:39.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:09:38.834+0000 7f0414de4000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T15:09:39.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:09:38.994+0000 7f0414de4000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T15:09:39.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:39 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:09:39.076+0000 7f0414de4000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T15:09:39.218 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:38 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:09:38.902+0000 7f7c615cb000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T15:09:39.218 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:39 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:09:39.064+0000 7f7c615cb000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T15:09:39.218 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:39 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:09:39.145+0000 7f7c615cb000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T15:09:39.718 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:39 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:09:39.454+0000 7f7c615cb000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T15:09:39.718 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:39 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:09:39.544+0000 7f7c615cb000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T15:09:39.718 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:39 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:09:39.635+0000 7f7c615cb000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T15:09:39.803 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:39 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:09:39.384+0000 7f0414de4000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T15:09:39.804 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:39 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:09:39.472+0000 7f0414de4000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T15:09:39.804 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:39 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:09:39.563+0000 7f0414de4000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T15:09:39.804 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:39 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:09:39.799+0000 7f0414de4000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T15:09:40.218 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:39 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:09:39.873+0000 7f7c615cb000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T15:09:40.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:09:40.230+0000 7f0414de4000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T15:09:40.718 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:40 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:09:40.308+0000 7f7c615cb000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T15:09:41.093 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:09:40.708+0000 7f0414de4000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T15:09:41.093 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:09:40.800+0000 7f0414de4000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T15:09:41.093 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:09:41.089+0000 7f0414de4000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T15:09:41.172 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:40 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:09:40.788+0000 7f7c615cb000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T15:09:41.172 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:40 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:09:40.880+0000 7f7c615cb000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T15:09:41.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:09:41.178+0000 7f0414de4000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T15:09:41.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:41 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:09:41.168+0000 7f7c615cb000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T15:09:41.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:41 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:09:41.255+0000 7f7c615cb000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T15:09:41.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:09:41.575+0000 7f0414de4000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T15:09:41.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:09:41.705+0000 7f0414de4000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T15:09:41.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:09:41.795+0000 7f0414de4000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T15:09:41.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:41 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:09:41.653+0000 7f7c615cb000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T15:09:41.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:41 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:09:41.783+0000 7f7c615cb000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T15:09:41.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:41 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:09:41.873+0000 7f7c615cb000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T15:09:42.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:09:41.909+0000 7f0414de4000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T15:09:42.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:09:41.996+0000 7f0414de4000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T15:09:42.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:41 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:09:41.986+0000 7f7c615cb000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T15:09:42.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:42 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:09:42.073+0000 7f7c615cb000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T15:09:42.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:42 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:09:42.536+0000 7f0414de4000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T15:09:42.957 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:42 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:09:42.612+0000 7f7c615cb000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T15:09:43.153 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:42 smithi145 ceph-mon[21899]: Active manager daemon y restarted 2024-04-22T15:09:43.154 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:42 smithi145 ceph-mon[21899]: Activating manager daemon y 2024-04-22T15:09:43.154 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:42 smithi145 ceph-mon[21899]: osdmap e60: 8 total, 8 up, 8 in 2024-04-22T15:09:43.154 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:42 smithi145 ceph-mon[26606]: Active manager daemon y restarted 2024-04-22T15:09:43.154 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:42 smithi145 ceph-mon[26606]: Activating manager daemon y 2024-04-22T15:09:43.154 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:42 smithi145 ceph-mon[26606]: osdmap e60: 8 total, 8 up, 8 in 2024-04-22T15:09:43.154 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:42 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:09:42.892+0000 7f0414de4000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T15:09:43.154 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: [22/Apr/2024:15:09:43] ENGINE Bus STARTING 2024-04-22T15:09:43.155 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: CherryPy Checker: 2024-04-22T15:09:43.155 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: The Application mounted at '' has an empty config. 2024-04-22T15:09:43.155 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:09:43.155 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: [22/Apr/2024:15:09:43] ENGINE Serving on http://:::9283 2024-04-22T15:09:43.155 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: [22/Apr/2024:15:09:43] ENGINE Bus STARTED 2024-04-22T15:09:43.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:42 smithi172 ceph-mon[22747]: Active manager daemon y restarted 2024-04-22T15:09:43.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:42 smithi172 ceph-mon[22747]: Activating manager daemon y 2024-04-22T15:09:43.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:42 smithi172 ceph-mon[22747]: osdmap e60: 8 total, 8 up, 8 in 2024-04-22T15:09:43.218 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:42 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:09:42.975+0000 7f7c615cb000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T15:09:43.218 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:42 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: [22/Apr/2024:15:09:42] ENGINE Bus STARTING 2024-04-22T15:09:43.218 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: CherryPy Checker: 2024-04-22T15:09:43.218 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: The Application mounted at '' has an empty config. 2024-04-22T15:09:43.218 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: 2024-04-22T15:09:43.219 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: [22/Apr/2024:15:09:43] ENGINE Serving on http://:::9283 2024-04-22T15:09:43.219 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: [22/Apr/2024:15:09:43] ENGINE Bus STARTED 2024-04-22T15:09:43.984 INFO:teuthology.orchestra.run.smithi172.stdout:Scheduled node-exporter update... 2024-04-22T15:09:44.049 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: [22/Apr/2024:15:09:43] ENGINE Bus STARTING 2024-04-22T15:09:44.049 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:44 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: [22/Apr/2024:15:09:44] ENGINE Serving on https://172.21.15.145:7150 2024-04-22T15:09:44.049 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[26606]: mgrmap e17: y(active, starting, since 0.0575266s), standbys: x 2024-04-22T15:09:44.049 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:09:44.049 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:09:44.049 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:09:44.049 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T15:09:44.050 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T15:09:44.050 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:09:44.050 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:09:44.050 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:09:44.050 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:09:44.050 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:09:44.050 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:09:44.050 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:09:44.050 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:09:44.050 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:09:44.050 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:09:44.050 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:09:44.050 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[26606]: Manager daemon y is now available 2024-04-22T15:09:44.051 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[26606]: Standby manager daemon x restarted 2024-04-22T15:09:44.051 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[26606]: Standby manager daemon x started 2024-04-22T15:09:44.051 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.172:0/1475805812' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T15:09:44.051 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.172:0/1475805812' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:09:44.051 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.172:0/1475805812' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T15:09:44.051 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:09:44.051 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.172:0/1475805812' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:09:44.051 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:09:44.051 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:44.051 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:09:44.051 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:44.051 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:09:44.051 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:09:44.052 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[21899]: mgrmap e17: y(active, starting, since 0.0575266s), standbys: x 2024-04-22T15:09:44.052 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:09:44.052 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:09:44.052 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:09:44.052 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T15:09:44.052 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T15:09:44.052 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:09:44.052 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:09:44.052 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:09:44.052 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:09:44.052 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:09:44.053 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:09:44.053 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:09:44.053 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:09:44.053 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:09:44.053 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:09:44.053 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:09:44.053 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[21899]: Manager daemon y is now available 2024-04-22T15:09:44.053 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[21899]: Standby manager daemon x restarted 2024-04-22T15:09:44.053 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[21899]: Standby manager daemon x started 2024-04-22T15:09:44.053 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.172:0/1475805812' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T15:09:44.053 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.172:0/1475805812' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:09:44.053 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.172:0/1475805812' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T15:09:44.054 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:09:44.054 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.172:0/1475805812' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:09:44.054 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:09:44.054 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:44.054 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:09:44.054 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:44.054 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:09:44.054 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:09:44.101 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-mon[22747]: mgrmap e17: y(active, starting, since 0.0575266s), standbys: x 2024-04-22T15:09:44.101 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:09:44.101 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:09:44.101 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:09:44.101 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T15:09:44.102 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T15:09:44.102 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:09:44.102 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:09:44.102 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:09:44.102 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:09:44.102 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:09:44.102 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:09:44.102 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:09:44.102 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:09:44.102 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:09:44.102 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:09:44.102 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:09:44.102 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-mon[22747]: Manager daemon y is now available 2024-04-22T15:09:44.102 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-mon[22747]: Standby manager daemon x restarted 2024-04-22T15:09:44.103 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-mon[22747]: Standby manager daemon x started 2024-04-22T15:09:44.103 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.172:0/1475805812' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T15:09:44.103 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.172:0/1475805812' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:09:44.103 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.172:0/1475805812' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T15:09:44.103 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:09:44.103 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.172:0/1475805812' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:09:44.103 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:09:44.103 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:09:44.103 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:09:44.103 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:44.103 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:09:44.103 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:09:44.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:09:44 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: [22/Apr/2024:15:09:44] ENGINE Bus STARTED 2024-04-22T15:09:44.694 DEBUG:teuthology.orchestra.run.smithi145:node-exporter.a> sudo journalctl -f -n 0 -u ceph-869afcfa-00b9-11ef-bc93-c7b262605968@node-exporter.a.service 2024-04-22T15:09:44.697 DEBUG:teuthology.orchestra.run.smithi172:node-exporter.b> sudo journalctl -f -n 0 -u ceph-869afcfa-00b9-11ef-bc93-c7b262605968@node-exporter.b.service 2024-04-22T15:09:44.699 INFO:tasks.cephadm:Adding alertmanager.a on smithi145 2024-04-22T15:09:44.700 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph orch apply alertmanager '1;smithi145=a' 2024-04-22T15:09:45.229 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:44 smithi145 ceph-mon[21899]: [22/Apr/2024:15:09:43] ENGINE Bus STARTING 2024-04-22T15:09:45.229 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:44 smithi145 ceph-mon[21899]: mgrmap e18: y(active, since 1.07461s), standbys: x 2024-04-22T15:09:45.230 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:44 smithi145 ceph-mon[21899]: from='client.24347 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi145=a;smithi172=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:09:45.230 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:44 smithi145 ceph-mon[21899]: Saving service node-exporter spec with placement smithi145=a;smithi172=b;count:2 2024-04-22T15:09:45.230 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:44 smithi145 ceph-mon[21899]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:45.230 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:44 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:45.230 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:44 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:45.230 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:44 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:45.230 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:44 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:45.230 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:44 smithi145 ceph-mon[26606]: [22/Apr/2024:15:09:43] ENGINE Bus STARTING 2024-04-22T15:09:45.230 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:44 smithi145 ceph-mon[26606]: mgrmap e18: y(active, since 1.07461s), standbys: x 2024-04-22T15:09:45.230 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:44 smithi145 ceph-mon[26606]: from='client.24347 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi145=a;smithi172=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:09:45.230 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:44 smithi145 ceph-mon[26606]: Saving service node-exporter spec with placement smithi145=a;smithi172=b;count:2 2024-04-22T15:09:45.231 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:44 smithi145 ceph-mon[26606]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:45.231 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:44 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:45.231 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:44 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:45.231 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:44 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:45.231 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:44 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:45.264 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:44 smithi172 ceph-mon[22747]: [22/Apr/2024:15:09:43] ENGINE Bus STARTING 2024-04-22T15:09:45.264 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:44 smithi172 ceph-mon[22747]: mgrmap e18: y(active, since 1.07461s), standbys: x 2024-04-22T15:09:45.264 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:44 smithi172 ceph-mon[22747]: from='client.24347 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi145=a;smithi172=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:09:45.264 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:44 smithi172 ceph-mon[22747]: Saving service node-exporter spec with placement smithi145=a;smithi172=b;count:2 2024-04-22T15:09:45.264 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:44 smithi172 ceph-mon[22747]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:45.264 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:44 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:45.264 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:44 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:45.264 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:44 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:45.264 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:44 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:46.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:45 smithi172 ceph-mon[22747]: [22/Apr/2024:15:09:44] ENGINE Serving on https://172.21.15.145:7150 2024-04-22T15:09:46.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:45 smithi172 ceph-mon[22747]: [22/Apr/2024:15:09:44] ENGINE Bus STARTED 2024-04-22T15:09:46.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:45 smithi172 ceph-mon[22747]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:46.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:45 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:46.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:45 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:46.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:45 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:46.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:45 smithi172 ceph-mon[22747]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-22T15:09:46.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:45 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:46.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:45 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:46.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:45 smithi145 ceph-mon[26606]: [22/Apr/2024:15:09:44] ENGINE Serving on https://172.21.15.145:7150 2024-04-22T15:09:46.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:45 smithi145 ceph-mon[26606]: [22/Apr/2024:15:09:44] ENGINE Bus STARTED 2024-04-22T15:09:46.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:45 smithi145 ceph-mon[26606]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:46.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:45 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:46.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:45 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:46.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:45 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:46.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:45 smithi145 ceph-mon[26606]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-22T15:09:46.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:45 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:46.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:45 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:46.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:45 smithi145 ceph-mon[21899]: [22/Apr/2024:15:09:44] ENGINE Serving on https://172.21.15.145:7150 2024-04-22T15:09:46.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:45 smithi145 ceph-mon[21899]: [22/Apr/2024:15:09:44] ENGINE Bus STARTED 2024-04-22T15:09:46.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:45 smithi145 ceph-mon[21899]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:46.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:45 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:46.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:45 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:46.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:45 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:46.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:45 smithi145 ceph-mon[21899]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-22T15:09:46.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:45 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:46.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:45 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:46.814 INFO:teuthology.orchestra.run.smithi172.stdout:Scheduled alertmanager update... 2024-04-22T15:09:47.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:46 smithi172 ceph-mon[22747]: Updating smithi145:/etc/ceph/ceph.conf 2024-04-22T15:09:47.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:46 smithi172 ceph-mon[22747]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:09:47.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:46 smithi172 ceph-mon[22747]: Updating smithi172:/etc/ceph/ceph.conf 2024-04-22T15:09:47.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:46 smithi172 ceph-mon[22747]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:09:47.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:46 smithi172 ceph-mon[22747]: Deploying daemon prometheus.a on smithi172 2024-04-22T15:09:47.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:46 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:47.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:46 smithi145 ceph-mon[21899]: Updating smithi145:/etc/ceph/ceph.conf 2024-04-22T15:09:47.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:46 smithi145 ceph-mon[21899]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:09:47.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:46 smithi145 ceph-mon[21899]: Updating smithi172:/etc/ceph/ceph.conf 2024-04-22T15:09:47.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:46 smithi145 ceph-mon[21899]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:09:47.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:46 smithi145 ceph-mon[21899]: Deploying daemon prometheus.a on smithi172 2024-04-22T15:09:47.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:46 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:47.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:46 smithi145 ceph-mon[26606]: Updating smithi145:/etc/ceph/ceph.conf 2024-04-22T15:09:47.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:46 smithi145 ceph-mon[26606]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:09:47.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:46 smithi145 ceph-mon[26606]: Updating smithi172:/etc/ceph/ceph.conf 2024-04-22T15:09:47.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:46 smithi145 ceph-mon[26606]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:09:47.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:46 smithi145 ceph-mon[26606]: Deploying daemon prometheus.a on smithi172 2024-04-22T15:09:47.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:46 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:47.404 DEBUG:teuthology.orchestra.run.smithi145:alertmanager.a> sudo journalctl -f -n 0 -u ceph-869afcfa-00b9-11ef-bc93-c7b262605968@alertmanager.a.service 2024-04-22T15:09:47.407 INFO:tasks.cephadm:Adding grafana.a on smithi172 2024-04-22T15:09:47.407 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph orch apply grafana '1;smithi172=a' 2024-04-22T15:09:48.263 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:47 smithi172 ceph-mon[22747]: from='client.24368 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi145=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:09:48.264 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:47 smithi172 ceph-mon[22747]: Saving service alertmanager spec with placement smithi145=a;count:1 2024-04-22T15:09:48.264 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:47 smithi172 ceph-mon[22747]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:48.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:47 smithi145 ceph-mon[26606]: from='client.24368 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi145=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:09:48.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:47 smithi145 ceph-mon[26606]: Saving service alertmanager spec with placement smithi145=a;count:1 2024-04-22T15:09:48.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:47 smithi145 ceph-mon[26606]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:48.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:47 smithi145 ceph-mon[21899]: from='client.24368 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi145=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:09:48.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:47 smithi145 ceph-mon[21899]: Saving service alertmanager spec with placement smithi145=a;count:1 2024-04-22T15:09:48.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:47 smithi145 ceph-mon[21899]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:50.095 INFO:teuthology.orchestra.run.smithi172.stdout:Scheduled grafana update... 2024-04-22T15:09:50.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:50 smithi172 ceph-mon[22747]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:50.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:50 smithi145 ceph-mon[21899]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:50.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:50 smithi145 ceph-mon[26606]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:51.050 DEBUG:teuthology.orchestra.run.smithi172:grafana.a> sudo journalctl -f -n 0 -u ceph-869afcfa-00b9-11ef-bc93-c7b262605968@grafana.a.service 2024-04-22T15:09:51.052 INFO:tasks.cephadm:Setting up client nodes... 2024-04-22T15:09:51.054 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-22T15:09:51.365 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:51 smithi145 ceph-mon[26606]: from='client.24374 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi172=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:09:51.365 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:51 smithi145 ceph-mon[26606]: Saving service grafana spec with placement smithi172=a;count:1 2024-04-22T15:09:51.365 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:51 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:51.365 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:51 smithi145 ceph-mon[26606]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:51.365 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:51 smithi145 ceph-mon[21899]: from='client.24374 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi172=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:09:51.365 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:51 smithi145 ceph-mon[21899]: Saving service grafana spec with placement smithi172=a;count:1 2024-04-22T15:09:51.365 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:51 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:51.366 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:51 smithi145 ceph-mon[21899]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:51.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:51 smithi172 ceph-mon[22747]: from='client.24374 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi172=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:09:51.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:51 smithi172 ceph-mon[22747]: Saving service grafana spec with placement smithi172=a;count:1 2024-04-22T15:09:51.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:51 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:09:51.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:51 smithi172 ceph-mon[22747]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:53.131 INFO:teuthology.orchestra.run.smithi145.stdout:[client.0] 2024-04-22T15:09:53.131 INFO:teuthology.orchestra.run.smithi145.stdout: key = AQDBfSZmZ39PBxAAq/v0rOya80E2JdJ+NWH2Zg== 2024-04-22T15:09:53.761 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-04-22T15:09:53.762 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-22T15:09:53.762 DEBUG:teuthology.orchestra.run.smithi145:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-22T15:09:53.799 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-22T15:09:54.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:54 smithi172 ceph-mon[22747]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:54.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:54 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/411820235' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T15:09:54.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:54 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/411820235' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-22T15:09:54.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:54 smithi145 ceph-mon[21899]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:54.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:54 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/411820235' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T15:09:54.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:54 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/411820235' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-22T15:09:54.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:54 smithi145 ceph-mon[26606]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:54.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:54 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/411820235' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T15:09:54.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:54 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/411820235' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-22T15:09:56.289 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:56 smithi172 ceph-mon[22747]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:56.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:56 smithi145 ceph-mon[21899]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:56.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:56 smithi145 ceph-mon[26606]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:57.138 INFO:teuthology.orchestra.run.smithi172.stdout:[client.1] 2024-04-22T15:09:57.138 INFO:teuthology.orchestra.run.smithi172.stdout: key = AQDFfSZmh6GpBxAACMeGIWScLCDWFfeosyZfbw== 2024-04-22T15:09:58.289 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:58 smithi172 ceph-mon[22747]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:58.289 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:58 smithi172 ceph-mon[22747]: from='client.? 172.21.15.172:0/2319635208' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T15:09:58.289 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:58 smithi172 ceph-mon[22747]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T15:09:58.290 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:09:58 smithi172 ceph-mon[22747]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-22T15:09:58.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:58 smithi145 ceph-mon[21899]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:58.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:58 smithi145 ceph-mon[21899]: from='client.? 172.21.15.172:0/2319635208' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T15:09:58.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:58 smithi145 ceph-mon[21899]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T15:09:58.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:09:58 smithi145 ceph-mon[21899]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-22T15:09:58.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:58 smithi145 ceph-mon[26606]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:09:58.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:58 smithi145 ceph-mon[26606]: from='client.? 172.21.15.172:0/2319635208' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T15:09:58.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:58 smithi145 ceph-mon[26606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T15:09:58.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:09:58 smithi145 ceph-mon[26606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-22T15:09:59.968 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:09:59 smithi172 systemd[1]: Starting Ceph prometheus.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:10:00.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:00 smithi145 ceph-mon[21899]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:00.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:00 smithi145 ceph-mon[21899]: overall HEALTH_OK 2024-04-22T15:10:00.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:00 smithi145 ceph-mon[26606]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:00.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:00 smithi145 ceph-mon[26606]: overall HEALTH_OK 2024-04-22T15:10:00.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:00 smithi172 ceph-mon[22747]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:00.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:00 smithi172 ceph-mon[22747]: overall HEALTH_OK 2024-04-22T15:10:01.262 DEBUG:teuthology.orchestra.run.smithi172:> set -ex 2024-04-22T15:10:01.262 DEBUG:teuthology.orchestra.run.smithi172:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-04-22T15:10:01.262 DEBUG:teuthology.orchestra.run.smithi172:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-04-22T15:10:01.304 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-22T15:10:01.305 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-22T15:10:01.305 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph mgr dump --format=json 2024-04-22T15:10:01.505 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:10:01 smithi172 podman[37871]: 2024-04-22 15:10:01.314350108 +0000 UTC m=+0.021279820 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-22T15:10:01.506 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:10:01 smithi172 podman[37871]: 2024-04-22 15:10:01.436536689 +0000 UTC m=+0.143466389 container create 6e86e365411f2b6c450ef7669e85c3fa26262d5408830e7e99d76d2d4237bde2 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:10:01.526 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T15:10:01.790 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:10:01 smithi172 podman[37871]: 2024-04-22 15:10:01.639084183 +0000 UTC m=+0.346013883 container init 6e86e365411f2b6c450ef7669e85c3fa26262d5408830e7e99d76d2d4237bde2 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:10:01.790 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:10:01 smithi172 podman[37871]: 2024-04-22 15:10:01.642410728 +0000 UTC m=+0.349340430 container start 6e86e365411f2b6c450ef7669e85c3fa26262d5408830e7e99d76d2d4237bde2 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:10:01.790 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:10:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[37887]: ts=2024-04-22T15:10:01.667Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-22T15:10:01.790 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:10:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[37887]: ts=2024-04-22T15:10:01.667Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-22T15:10:01.790 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:10:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[37887]: ts=2024-04-22T15:10:01.667Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-22T15:10:01.790 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:10:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[37887]: ts=2024-04-22T15:10:01.667Z caller=main.go:518 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi172 (none))" 2024-04-22T15:10:01.790 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:10:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[37887]: ts=2024-04-22T15:10:01.667Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-22T15:10:01.790 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:10:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[37887]: ts=2024-04-22T15:10:01.667Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-22T15:10:01.790 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:10:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[37887]: ts=2024-04-22T15:10:01.668Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-22T15:10:01.791 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:10:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[37887]: ts=2024-04-22T15:10:01.668Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-22T15:10:01.791 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:10:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[37887]: ts=2024-04-22T15:10:01.670Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-22T15:10:01.791 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:10:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[37887]: ts=2024-04-22T15:10:01.672Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-22T15:10:01.791 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:10:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[37887]: ts=2024-04-22T15:10:01.672Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.711µs 2024-04-22T15:10:01.791 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:10:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[37887]: ts=2024-04-22T15:10:01.672Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-22T15:10:01.791 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:10:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[37887]: ts=2024-04-22T15:10:01.672Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-04-22T15:10:01.791 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:10:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[37887]: ts=2024-04-22T15:10:01.672Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=20.311µs wal_replay_duration=145.365µs total_replay_duration=180.376µs 2024-04-22T15:10:01.791 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:10:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[37887]: ts=2024-04-22T15:10:01.673Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-22T15:10:01.791 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:10:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[37887]: ts=2024-04-22T15:10:01.673Z caller=main.go:947 level=info msg="TSDB started" 2024-04-22T15:10:01.791 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:10:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[37887]: ts=2024-04-22T15:10:01.673Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-22T15:10:01.791 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:10:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[37887]: ts=2024-04-22T15:10:01.688Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.196986ms db_storage=487ns remote_storage=1.196µs web_handler=200ns query_engine=581ns scrape=397.571µs scrape_sd=15.287µs notify=835ns notify_sd=1.384µs rules=14.607378ms 2024-04-22T15:10:01.791 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:10:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[37887]: ts=2024-04-22T15:10:01.688Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-22T15:10:01.791 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:10:01 smithi172 bash[37871]: 6e86e365411f2b6c450ef7669e85c3fa26262d5408830e7e99d76d2d4237bde2 2024-04-22T15:10:01.791 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:10:01 smithi172 systemd[1]: Started Ceph prometheus.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:10:02.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:02 smithi172 ceph-mon[22747]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:02.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:02 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:02.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:02 smithi145 ceph-mon[21899]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:02.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:02 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:02.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:02 smithi145 ceph-mon[26606]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:02.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:02 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:03.237 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:03 smithi145 ceph-mon[26606]: Deploying daemon node-exporter.a on smithi145 2024-04-22T15:10:03.238 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:03 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:03.238 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:03 smithi145 ceph-mon[21899]: Deploying daemon node-exporter.a on smithi145 2024-04-22T15:10:03.238 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:03 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:03.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:03 smithi172 ceph-mon[22747]: Deploying daemon node-exporter.a on smithi145 2024-04-22T15:10:03.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:03 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:03.592 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-04-22T15:10:03.873 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:03 smithi145 systemd[1]: Starting Ceph node-exporter.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:10:04.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:04 smithi145 ceph-mon[26606]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:04.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:04 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/1346410967' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-22T15:10:04.209 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:04 smithi145 ceph-mon[21899]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:04.209 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:04 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/1346410967' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-22T15:10:04.234 INFO:teuthology.orchestra.run.smithi145.stdout:{"epoch":19,"active_gid":14448,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6800","nonce":3013561970},{"type":"v1","addr":"172.21.15.145:6801","nonce":3013561970}]},"active_addr":"172.21.15.145:6801/3013561970","active_change":"2024-04-22T15:09:42.895166+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":24335,"name":"x","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.145:8443/","prometheus":"http://172.21.15.145:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":60,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.145:0","nonce":3433168494}]},{"addrvec":[{"type":"v2","addr":"172.21.15.145:0","nonce":2937248060}]},{"addrvec":[{"type":"v2","addr":"172.21.15.145:0","nonce":1768331515}]},{"addrvec":[{"type":"v2","addr":"172.21.15.145:0","nonce":2469616922}]}]}} 2024-04-22T15:10:04.241 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-22T15:10:04.241 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-22T15:10:04.241 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-22T15:10:04.466 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T15:10:04.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:04 smithi172 ceph-mon[22747]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:04.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:04 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/1346410967' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-22T15:10:04.589 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:04 smithi145 bash[41500]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-04-22T15:10:05.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:05 smithi145 ceph-mon[21899]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:05.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:05 smithi145 ceph-mon[26606]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:05.124 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:04 smithi145 bash[41500]: Getting image source signatures 2024-04-22T15:10:05.124 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:04 smithi145 bash[41500]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-04-22T15:10:05.124 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:04 smithi145 bash[41500]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-04-22T15:10:05.124 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:04 smithi145 bash[41500]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-04-22T15:10:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:05 smithi172 ceph-mon[22747]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:07.112 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-04-22T15:10:07.113 INFO:teuthology.orchestra.run.smithi145.stdout:{"epoch":60,"fsid":"869afcfa-00b9-11ef-bc93-c7b262605968","created":"2024-04-22T15:04:33.942175+0000","modified":"2024-04-22T15:09:42.894798+0000","last_up_change":"2024-04-22T15:09:23.783719+0000","last_in_change":"2024-04-22T15:09:06.672949+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-22T15:07:30.182778+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"8f61de15-c199-4966-8d5d-8acd8ca2d6c8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6802","nonce":2466746205},{"type":"v1","addr":"172.21.15.145:6803","nonce":2466746205}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6804","nonce":2466746205},{"type":"v1","addr":"172.21.15.145:6805","nonce":2466746205}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6808","nonce":2466746205},{"type":"v1","addr":"172.21.15.145:6809","nonce":2466746205}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6806","nonce":2466746205},{"type":"v1","addr":"172.21.15.145:6807","nonce":2466746205}]},"public_addr":"172.21.15.145:6803/2466746205","cluster_addr":"172.21.15.145:6805/2466746205","heartbeat_back_addr":"172.21.15.145:6809/2466746205","heartbeat_front_addr":"172.21.15.145:6807/2466746205","state":["exists","up"]},{"osd":1,"uuid":"f863b9ae-7cb4-4b74-a2a1-a2384e809318","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6810","nonce":1586792147},{"type":"v1","addr":"172.21.15.145:6811","nonce":1586792147}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6812","nonce":1586792147},{"type":"v1","addr":"172.21.15.145:6813","nonce":1586792147}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6816","nonce":1586792147},{"type":"v1","addr":"172.21.15.145:6817","nonce":1586792147}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6814","nonce":1586792147},{"type":"v1","addr":"172.21.15.145:6815","nonce":1586792147}]},"public_addr":"172.21.15.145:6811/1586792147","cluster_addr":"172.21.15.145:6813/1586792147","heartbeat_back_addr":"172.21.15.145:6817/1586792147","heartbeat_front_addr":"172.21.15.145:6815/1586792147","state":["exists","up"]},{"osd":2,"uuid":"359c8817-27db-4026-891b-909210f255bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6818","nonce":3315449646},{"type":"v1","addr":"172.21.15.145:6819","nonce":3315449646}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6820","nonce":3315449646},{"type":"v1","addr":"172.21.15.145:6821","nonce":3315449646}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6824","nonce":3315449646},{"type":"v1","addr":"172.21.15.145:6825","nonce":3315449646}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6822","nonce":3315449646},{"type":"v1","addr":"172.21.15.145:6823","nonce":3315449646}]},"public_addr":"172.21.15.145:6819/3315449646","cluster_addr":"172.21.15.145:6821/3315449646","heartbeat_back_addr":"172.21.15.145:6825/3315449646","heartbeat_front_addr":"172.21.15.145:6823/3315449646","state":["exists","up"]},{"osd":3,"uuid":"25f5ce78-1153-489a-bdc9-7cbd4ff4cfe5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6826","nonce":922673347},{"type":"v1","addr":"172.21.15.145:6827","nonce":922673347}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6828","nonce":922673347},{"type":"v1","addr":"172.21.15.145:6829","nonce":922673347}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6832","nonce":922673347},{"type":"v1","addr":"172.21.15.145:6833","nonce":922673347}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6830","nonce":922673347},{"type":"v1","addr":"172.21.15.145:6831","nonce":922673347}]},"public_addr":"172.21.15.145:6827/922673347","cluster_addr":"172.21.15.145:6829/922673347","heartbeat_back_addr":"172.21.15.145:6833/922673347","heartbeat_front_addr":"172.21.15.145:6831/922673347","state":["exists","up"]},{"osd":4,"uuid":"eefb46ad-39a3-40bf-a966-65f7a32841f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6800","nonce":3136053699},{"type":"v1","addr":"172.21.15.172:6801","nonce":3136053699}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6802","nonce":3136053699},{"type":"v1","addr":"172.21.15.172:6803","nonce":3136053699}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6806","nonce":3136053699},{"type":"v1","addr":"172.21.15.172:6807","nonce":3136053699}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6804","nonce":3136053699},{"type":"v1","addr":"172.21.15.172:6805","nonce":3136053699}]},"public_addr":"172.21.15.172:6801/3136053699","cluster_addr":"172.21.15.172:6803/3136053699","heartbeat_back_addr":"172.21.15.172:6807/3136053699","heartbeat_front_addr":"172.21.15.172:6805/3136053699","state":["exists","up"]},{"osd":5,"uuid":"58c9f92d-ff6c-4774-a08a-e2377ea0e95f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6808","nonce":878495},{"type":"v1","addr":"172.21.15.172:6809","nonce":878495}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6810","nonce":878495},{"type":"v1","addr":"172.21.15.172:6811","nonce":878495}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6814","nonce":878495},{"type":"v1","addr":"172.21.15.172:6815","nonce":878495}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6812","nonce":878495},{"type":"v1","addr":"172.21.15.172:6813","nonce":878495}]},"public_addr":"172.21.15.172:6809/878495","cluster_addr":"172.21.15.172:6811/878495","heartbeat_back_addr":"172.21.15.172:6815/878495","heartbeat_front_addr":"172.21.15.172:6813/878495","state":["exists","up"]},{"osd":6,"uuid":"ff30003d-cc8d-4e7b-beb0-56a4813be817","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6816","nonce":2239493705},{"type":"v1","addr":"172.21.15.172:6817","nonce":2239493705}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6818","nonce":2239493705},{"type":"v1","addr":"172.21.15.172:6819","nonce":2239493705}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6822","nonce":2239493705},{"type":"v1","addr":"172.21.15.172:6823","nonce":2239493705}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6820","nonce":2239493705},{"type":"v1","addr":"172.21.15.172:6821","nonce":2239493705}]},"public_addr":"172.21.15.172:6817/2239493705","cluster_addr":"172.21.15.172:6819/2239493705","heartbeat_back_addr":"172.21.15.172:6823/2239493705","heartbeat_front_addr":"172.21.15.172:6821/2239493705","state":["exists","up"]},{"osd":7,"uuid":"7723e772-46df-4001-8f36-83b3695c9465","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6824","nonce":2431704909},{"type":"v1","addr":"172.21.15.172:6825","nonce":2431704909}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6826","nonce":2431704909},{"type":"v1","addr":"172.21.15.172:6827","nonce":2431704909}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6830","nonce":2431704909},{"type":"v1","addr":"172.21.15.172:6831","nonce":2431704909}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6828","nonce":2431704909},{"type":"v1","addr":"172.21.15.172:6829","nonce":2431704909}]},"public_addr":"172.21.15.172:6825/2431704909","cluster_addr":"172.21.15.172:6827/2431704909","heartbeat_back_addr":"172.21.15.172:6831/2431704909","heartbeat_front_addr":"172.21.15.172:6829/2431704909","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:06:39.225018+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:07:03.246108+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:07:26.590807+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:07:51.641916+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:08:13.263422+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:08:35.886011+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:08:58.567855+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:09:21.846973+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.145:6801/3974605882":"2024-04-23T15:09:42.894767+0000","172.21.15.145:0/3245087794":"2024-04-23T15:09:42.894767+0000","172.21.15.145:0/2226900857":"2024-04-23T15:05:19.220973+0000","172.21.15.145:0/2747862617":"2024-04-23T15:09:42.894767+0000","172.21.15.145:6800/3974605882":"2024-04-23T15:09:42.894767+0000","172.21.15.145:6801/1271607645":"2024-04-23T15:05:19.220973+0000","172.21.15.145:0/1075985196":"2024-04-23T15:04:56.813632+0000","172.21.15.145:0/3793806840":"2024-04-23T15:04:56.813632+0000","172.21.15.145:0/1268350667":"2024-04-23T15:09:42.894767+0000","172.21.15.145:0/1261892263":"2024-04-23T15:04:56.813632+0000","172.21.15.145:0/1333376048":"2024-04-23T15:05:19.220973+0000","172.21.15.145:6800/4174602953":"2024-04-23T15:04:56.813632+0000","172.21.15.145:0/2791239557":"2024-04-23T15:09:42.894767+0000","172.21.15.145:0/218026216":"2024-04-23T15:05:19.220973+0000","172.21.15.145:6800/1271607645":"2024-04-23T15:05:19.220973+0000","172.21.15.145:6801/4174602953":"2024-04-23T15:04:56.813632+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-22T15:10:07.646 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:07 smithi145 bash[41500]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-04-22T15:10:08.016 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:07 smithi145 bash[41500]: Writing manifest to image destination 2024-04-22T15:10:08.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-mon[26606]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:08.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3364996484' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T15:10:08.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-mon[21899]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:08.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3364996484' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T15:10:08.309 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 podman[41500]: 2024-04-22 15:10:08.182421656 +0000 UTC m=+3.885357991 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-04-22T15:10:08.337 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-22T15:10:08.337 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-22T15:10:08.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:08 smithi172 ceph-mon[22747]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:08.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:08 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/3364996484' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T15:10:08.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:10:07 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:10:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:10:08.560 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 podman[41500]: 2024-04-22 15:10:08.43915757 +0000 UTC m=+4.142093890 container create 5e2f432a509124c22637233b31a0632ea5d5942499c4a315e8159917d07bcc21 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T15:10:08.804 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T15:10:08.928 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 podman[41500]: 2024-04-22 15:10:08.657818292 +0000 UTC m=+4.360754612 container init 5e2f432a509124c22637233b31a0632ea5d5942499c4a315e8159917d07bcc21 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T15:10:08.928 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 podman[41500]: 2024-04-22 15:10:08.660537421 +0000 UTC m=+4.363473745 container start 5e2f432a509124c22637233b31a0632ea5d5942499c4a315e8159917d07bcc21 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T15:10:08.928 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.664Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-04-22T15:10:08.929 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.664Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-04-22T15:10:08.929 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.664Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-04-22T15:10:08.929 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.664Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-22T15:10:08.929 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-04-22T15:10:08.929 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=arp 2024-04-22T15:10:08.929 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=bcache 2024-04-22T15:10:08.929 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=bonding 2024-04-22T15:10:08.929 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=btrfs 2024-04-22T15:10:08.929 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=conntrack 2024-04-22T15:10:08.929 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=cpu 2024-04-22T15:10:08.929 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-04-22T15:10:08.929 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=diskstats 2024-04-22T15:10:08.929 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=dmi 2024-04-22T15:10:08.929 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=edac 2024-04-22T15:10:08.930 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=entropy 2024-04-22T15:10:08.930 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-04-22T15:10:08.930 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=filefd 2024-04-22T15:10:08.930 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=filesystem 2024-04-22T15:10:08.930 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=hwmon 2024-04-22T15:10:08.930 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=infiniband 2024-04-22T15:10:08.930 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=ipvs 2024-04-22T15:10:08.930 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=loadavg 2024-04-22T15:10:08.930 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=mdadm 2024-04-22T15:10:08.931 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=meminfo 2024-04-22T15:10:08.931 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=netclass 2024-04-22T15:10:08.931 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=netdev 2024-04-22T15:10:08.931 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=netstat 2024-04-22T15:10:08.931 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=nfs 2024-04-22T15:10:08.931 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=nfsd 2024-04-22T15:10:08.931 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=nvme 2024-04-22T15:10:08.931 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=os 2024-04-22T15:10:08.931 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-04-22T15:10:08.931 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=pressure 2024-04-22T15:10:08.931 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=rapl 2024-04-22T15:10:08.931 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=schedstat 2024-04-22T15:10:08.931 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=sockstat 2024-04-22T15:10:08.931 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=softnet 2024-04-22T15:10:08.931 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=stat 2024-04-22T15:10:08.932 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=tapestats 2024-04-22T15:10:08.932 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=textfile 2024-04-22T15:10:08.932 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-04-22T15:10:08.932 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=time 2024-04-22T15:10:08.932 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-04-22T15:10:08.932 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=uname 2024-04-22T15:10:08.932 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=vmstat 2024-04-22T15:10:08.932 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=xfs 2024-04-22T15:10:08.932 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:115 level=info collector=zfs 2024-04-22T15:10:08.932 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.665Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-04-22T15:10:08.932 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[41714]: ts=2024-04-22T15:10:08.666Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-04-22T15:10:08.932 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 bash[41500]: 5e2f432a509124c22637233b31a0632ea5d5942499c4a315e8159917d07bcc21 2024-04-22T15:10:08.932 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:10:08 smithi145 systemd[1]: Started Ceph node-exporter.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:10:09.457 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:09 smithi172 systemd[1]: Starting Ceph node-exporter.b for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:10:09.718 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:09 smithi172 bash[38129]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-04-22T15:10:10.119 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:09 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:10.119 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:09 smithi145 ceph-mon[26606]: Deploying daemon node-exporter.b on smithi172 2024-04-22T15:10:10.119 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:09 smithi145 ceph-mon[26606]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:10.119 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:09 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:10.120 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:09 smithi145 ceph-mon[21899]: Deploying daemon node-exporter.b on smithi172 2024-04-22T15:10:10.120 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:09 smithi145 ceph-mon[21899]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:10.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:09 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:10.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:09 smithi172 ceph-mon[22747]: Deploying daemon node-exporter.b on smithi172 2024-04-22T15:10:10.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:09 smithi172 ceph-mon[22747]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:10.218 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:10 smithi172 bash[38129]: Getting image source signatures 2024-04-22T15:10:10.218 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:10 smithi172 bash[38129]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-04-22T15:10:10.218 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:10 smithi172 bash[38129]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-04-22T15:10:10.219 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:10 smithi172 bash[38129]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-04-22T15:10:10.622 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-04-22T15:10:10.622 INFO:teuthology.orchestra.run.smithi145.stdout:{"epoch":60,"fsid":"869afcfa-00b9-11ef-bc93-c7b262605968","created":"2024-04-22T15:04:33.942175+0000","modified":"2024-04-22T15:09:42.894798+0000","last_up_change":"2024-04-22T15:09:23.783719+0000","last_in_change":"2024-04-22T15:09:06.672949+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-22T15:07:30.182778+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"8f61de15-c199-4966-8d5d-8acd8ca2d6c8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6802","nonce":2466746205},{"type":"v1","addr":"172.21.15.145:6803","nonce":2466746205}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6804","nonce":2466746205},{"type":"v1","addr":"172.21.15.145:6805","nonce":2466746205}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6808","nonce":2466746205},{"type":"v1","addr":"172.21.15.145:6809","nonce":2466746205}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6806","nonce":2466746205},{"type":"v1","addr":"172.21.15.145:6807","nonce":2466746205}]},"public_addr":"172.21.15.145:6803/2466746205","cluster_addr":"172.21.15.145:6805/2466746205","heartbeat_back_addr":"172.21.15.145:6809/2466746205","heartbeat_front_addr":"172.21.15.145:6807/2466746205","state":["exists","up"]},{"osd":1,"uuid":"f863b9ae-7cb4-4b74-a2a1-a2384e809318","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6810","nonce":1586792147},{"type":"v1","addr":"172.21.15.145:6811","nonce":1586792147}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6812","nonce":1586792147},{"type":"v1","addr":"172.21.15.145:6813","nonce":1586792147}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6816","nonce":1586792147},{"type":"v1","addr":"172.21.15.145:6817","nonce":1586792147}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6814","nonce":1586792147},{"type":"v1","addr":"172.21.15.145:6815","nonce":1586792147}]},"public_addr":"172.21.15.145:6811/1586792147","cluster_addr":"172.21.15.145:6813/1586792147","heartbeat_back_addr":"172.21.15.145:6817/1586792147","heartbeat_front_addr":"172.21.15.145:6815/1586792147","state":["exists","up"]},{"osd":2,"uuid":"359c8817-27db-4026-891b-909210f255bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6818","nonce":3315449646},{"type":"v1","addr":"172.21.15.145:6819","nonce":3315449646}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6820","nonce":3315449646},{"type":"v1","addr":"172.21.15.145:6821","nonce":3315449646}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6824","nonce":3315449646},{"type":"v1","addr":"172.21.15.145:6825","nonce":3315449646}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6822","nonce":3315449646},{"type":"v1","addr":"172.21.15.145:6823","nonce":3315449646}]},"public_addr":"172.21.15.145:6819/3315449646","cluster_addr":"172.21.15.145:6821/3315449646","heartbeat_back_addr":"172.21.15.145:6825/3315449646","heartbeat_front_addr":"172.21.15.145:6823/3315449646","state":["exists","up"]},{"osd":3,"uuid":"25f5ce78-1153-489a-bdc9-7cbd4ff4cfe5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6826","nonce":922673347},{"type":"v1","addr":"172.21.15.145:6827","nonce":922673347}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6828","nonce":922673347},{"type":"v1","addr":"172.21.15.145:6829","nonce":922673347}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6832","nonce":922673347},{"type":"v1","addr":"172.21.15.145:6833","nonce":922673347}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6830","nonce":922673347},{"type":"v1","addr":"172.21.15.145:6831","nonce":922673347}]},"public_addr":"172.21.15.145:6827/922673347","cluster_addr":"172.21.15.145:6829/922673347","heartbeat_back_addr":"172.21.15.145:6833/922673347","heartbeat_front_addr":"172.21.15.145:6831/922673347","state":["exists","up"]},{"osd":4,"uuid":"eefb46ad-39a3-40bf-a966-65f7a32841f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6800","nonce":3136053699},{"type":"v1","addr":"172.21.15.172:6801","nonce":3136053699}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6802","nonce":3136053699},{"type":"v1","addr":"172.21.15.172:6803","nonce":3136053699}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6806","nonce":3136053699},{"type":"v1","addr":"172.21.15.172:6807","nonce":3136053699}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6804","nonce":3136053699},{"type":"v1","addr":"172.21.15.172:6805","nonce":3136053699}]},"public_addr":"172.21.15.172:6801/3136053699","cluster_addr":"172.21.15.172:6803/3136053699","heartbeat_back_addr":"172.21.15.172:6807/3136053699","heartbeat_front_addr":"172.21.15.172:6805/3136053699","state":["exists","up"]},{"osd":5,"uuid":"58c9f92d-ff6c-4774-a08a-e2377ea0e95f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6808","nonce":878495},{"type":"v1","addr":"172.21.15.172:6809","nonce":878495}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6810","nonce":878495},{"type":"v1","addr":"172.21.15.172:6811","nonce":878495}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6814","nonce":878495},{"type":"v1","addr":"172.21.15.172:6815","nonce":878495}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6812","nonce":878495},{"type":"v1","addr":"172.21.15.172:6813","nonce":878495}]},"public_addr":"172.21.15.172:6809/878495","cluster_addr":"172.21.15.172:6811/878495","heartbeat_back_addr":"172.21.15.172:6815/878495","heartbeat_front_addr":"172.21.15.172:6813/878495","state":["exists","up"]},{"osd":6,"uuid":"ff30003d-cc8d-4e7b-beb0-56a4813be817","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6816","nonce":2239493705},{"type":"v1","addr":"172.21.15.172:6817","nonce":2239493705}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6818","nonce":2239493705},{"type":"v1","addr":"172.21.15.172:6819","nonce":2239493705}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6822","nonce":2239493705},{"type":"v1","addr":"172.21.15.172:6823","nonce":2239493705}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6820","nonce":2239493705},{"type":"v1","addr":"172.21.15.172:6821","nonce":2239493705}]},"public_addr":"172.21.15.172:6817/2239493705","cluster_addr":"172.21.15.172:6819/2239493705","heartbeat_back_addr":"172.21.15.172:6823/2239493705","heartbeat_front_addr":"172.21.15.172:6821/2239493705","state":["exists","up"]},{"osd":7,"uuid":"7723e772-46df-4001-8f36-83b3695c9465","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6824","nonce":2431704909},{"type":"v1","addr":"172.21.15.172:6825","nonce":2431704909}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6826","nonce":2431704909},{"type":"v1","addr":"172.21.15.172:6827","nonce":2431704909}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6830","nonce":2431704909},{"type":"v1","addr":"172.21.15.172:6831","nonce":2431704909}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6828","nonce":2431704909},{"type":"v1","addr":"172.21.15.172:6829","nonce":2431704909}]},"public_addr":"172.21.15.172:6825/2431704909","cluster_addr":"172.21.15.172:6827/2431704909","heartbeat_back_addr":"172.21.15.172:6831/2431704909","heartbeat_front_addr":"172.21.15.172:6829/2431704909","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:06:39.225018+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:07:03.246108+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:07:26.590807+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:07:51.641916+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:08:13.263422+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:08:35.886011+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:08:58.567855+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T15:09:21.846973+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.145:6801/3974605882":"2024-04-23T15:09:42.894767+0000","172.21.15.145:0/3245087794":"2024-04-23T15:09:42.894767+0000","172.21.15.145:0/2226900857":"2024-04-23T15:05:19.220973+0000","172.21.15.145:0/2747862617":"2024-04-23T15:09:42.894767+0000","172.21.15.145:6800/3974605882":"2024-04-23T15:09:42.894767+0000","172.21.15.145:6801/1271607645":"2024-04-23T15:05:19.220973+0000","172.21.15.145:0/1075985196":"2024-04-23T15:04:56.813632+0000","172.21.15.145:0/3793806840":"2024-04-23T15:04:56.813632+0000","172.21.15.145:0/1268350667":"2024-04-23T15:09:42.894767+0000","172.21.15.145:0/1261892263":"2024-04-23T15:04:56.813632+0000","172.21.15.145:0/1333376048":"2024-04-23T15:05:19.220973+0000","172.21.15.145:6800/4174602953":"2024-04-23T15:04:56.813632+0000","172.21.15.145:0/2791239557":"2024-04-23T15:09:42.894767+0000","172.21.15.145:0/218026216":"2024-04-23T15:05:19.220973+0000","172.21.15.145:6800/1271607645":"2024-04-23T15:05:19.220973+0000","172.21.15.145:6801/4174602953":"2024-04-23T15:04:56.813632+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-22T15:10:11.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:10 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3317935279' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T15:10:11.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:10 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3317935279' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T15:10:11.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:10 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/3317935279' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T15:10:11.335 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-22T15:10:11.336 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-22T15:10:11.336 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-22T15:10:11.336 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-22T15:10:11.337 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-22T15:10:11.338 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-22T15:10:11.338 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-22T15:10:11.338 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-22T15:10:11.718 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:11 smithi172 bash[38129]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-04-22T15:10:11.718 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:11 smithi172 bash[38129]: Writing manifest to image destination 2024-04-22T15:10:11.747 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T15:10:11.759 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T15:10:11.762 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T15:10:11.762 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T15:10:11.765 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T15:10:11.767 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T15:10:11.767 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T15:10:11.799 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T15:10:12.074 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:11 smithi172 podman[38129]: 2024-04-22 15:10:11.851106075 +0000 UTC m=+2.294454947 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-04-22T15:10:12.074 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:11 smithi172 podman[38129]: 2024-04-22 15:10:11.94975422 +0000 UTC m=+2.393103077 container create 759c7e0c3e41d5ed7ae0c268f94143807f31869084177f7b77c89c31eeaf141d (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T15:10:12.074 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:11 smithi172 ceph-mon[22747]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:12.075 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:11 smithi145 ceph-mon[26606]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:12.075 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:11 smithi145 ceph-mon[21899]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:12.075 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:10:11 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:10:11] "GET /metrics HTTP/1.1" 200 191514 "" "Prometheus/2.33.4" 2024-04-22T15:10:12.469 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 podman[38129]: 2024-04-22 15:10:12.176913426 +0000 UTC m=+2.620262284 container init 759c7e0c3e41d5ed7ae0c268f94143807f31869084177f7b77c89c31eeaf141d (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T15:10:12.469 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 podman[38129]: 2024-04-22 15:10:12.179882068 +0000 UTC m=+2.623230935 container start 759c7e0c3e41d5ed7ae0c268f94143807f31869084177f7b77c89c31eeaf141d (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T15:10:12.469 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.183Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-04-22T15:10:12.469 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.183Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-04-22T15:10:12.469 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.184Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-04-22T15:10:12.469 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.184Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-22T15:10:12.469 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.184Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-04-22T15:10:12.469 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.184Z caller=node_exporter.go:115 level=info collector=arp 2024-04-22T15:10:12.469 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.184Z caller=node_exporter.go:115 level=info collector=bcache 2024-04-22T15:10:12.470 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.184Z caller=node_exporter.go:115 level=info collector=bonding 2024-04-22T15:10:12.470 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.184Z caller=node_exporter.go:115 level=info collector=btrfs 2024-04-22T15:10:12.470 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.184Z caller=node_exporter.go:115 level=info collector=conntrack 2024-04-22T15:10:12.470 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.184Z caller=node_exporter.go:115 level=info collector=cpu 2024-04-22T15:10:12.470 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.184Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-04-22T15:10:12.470 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.184Z caller=node_exporter.go:115 level=info collector=diskstats 2024-04-22T15:10:12.470 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.184Z caller=node_exporter.go:115 level=info collector=dmi 2024-04-22T15:10:12.470 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.184Z caller=node_exporter.go:115 level=info collector=edac 2024-04-22T15:10:12.470 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.184Z caller=node_exporter.go:115 level=info collector=entropy 2024-04-22T15:10:12.470 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.184Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-04-22T15:10:12.470 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=node_exporter.go:115 level=info collector=filefd 2024-04-22T15:10:12.470 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=node_exporter.go:115 level=info collector=filesystem 2024-04-22T15:10:12.470 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=node_exporter.go:115 level=info collector=hwmon 2024-04-22T15:10:12.470 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=node_exporter.go:115 level=info collector=infiniband 2024-04-22T15:10:12.471 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=node_exporter.go:115 level=info collector=ipvs 2024-04-22T15:10:12.471 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=node_exporter.go:115 level=info collector=loadavg 2024-04-22T15:10:12.471 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=node_exporter.go:115 level=info collector=mdadm 2024-04-22T15:10:12.471 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=node_exporter.go:115 level=info collector=meminfo 2024-04-22T15:10:12.471 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=node_exporter.go:115 level=info collector=netclass 2024-04-22T15:10:12.471 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=node_exporter.go:115 level=info collector=netdev 2024-04-22T15:10:12.471 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=node_exporter.go:115 level=info collector=netstat 2024-04-22T15:10:12.471 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=node_exporter.go:115 level=info collector=nfs 2024-04-22T15:10:12.471 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=node_exporter.go:115 level=info collector=nfsd 2024-04-22T15:10:12.472 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=node_exporter.go:115 level=info collector=nvme 2024-04-22T15:10:12.472 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=node_exporter.go:115 level=info collector=os 2024-04-22T15:10:12.472 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-04-22T15:10:12.472 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=node_exporter.go:115 level=info collector=pressure 2024-04-22T15:10:12.472 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=node_exporter.go:115 level=info collector=rapl 2024-04-22T15:10:12.472 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=node_exporter.go:115 level=info collector=schedstat 2024-04-22T15:10:12.472 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=node_exporter.go:115 level=info collector=sockstat 2024-04-22T15:10:12.472 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=node_exporter.go:115 level=info collector=softnet 2024-04-22T15:10:12.472 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=node_exporter.go:115 level=info collector=stat 2024-04-22T15:10:12.472 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=node_exporter.go:115 level=info collector=tapestats 2024-04-22T15:10:12.472 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=node_exporter.go:115 level=info collector=textfile 2024-04-22T15:10:12.472 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-04-22T15:10:12.472 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=node_exporter.go:115 level=info collector=time 2024-04-22T15:10:12.473 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-04-22T15:10:12.473 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=node_exporter.go:115 level=info collector=uname 2024-04-22T15:10:12.473 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=node_exporter.go:115 level=info collector=vmstat 2024-04-22T15:10:12.473 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=node_exporter.go:115 level=info collector=xfs 2024-04-22T15:10:12.473 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=node_exporter.go:115 level=info collector=zfs 2024-04-22T15:10:12.473 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-04-22T15:10:12.473 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[38181]: ts=2024-04-22T15:10:12.185Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-04-22T15:10:12.473 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 bash[38129]: 759c7e0c3e41d5ed7ae0c268f94143807f31869084177f7b77c89c31eeaf141d 2024-04-22T15:10:12.473 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:10:12 smithi172 systemd[1]: Started Ceph node-exporter.b for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:10:13.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:13 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:13.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:13 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:10:13.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:13 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:10:13.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:13 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:10:13.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:13 smithi145 ceph-mon[21899]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:13.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:13 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:13.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:13 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:13.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:13 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:10:13.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:13 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:10:13.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:13 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:10:13.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:13 smithi145 ceph-mon[26606]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:13.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:13 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:13.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:13 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:13.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:13 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:10:13.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:13 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:10:13.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:13 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:10:13.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:13 smithi172 ceph-mon[22747]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:13.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:13 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:16.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:15 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:16.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:15 smithi172 ceph-mon[22747]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:16.243 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:15 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:16.244 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:15 smithi145 ceph-mon[26606]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:16.244 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:15 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:16.244 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:15 smithi145 ceph-mon[21899]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:18.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:18 smithi145 ceph-mon[26606]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:18.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:18 smithi145 ceph-mon[21899]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:18.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:18 smithi172 ceph-mon[22747]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:18.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:10:17 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:10:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:10:20.337 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:20 smithi145 ceph-mon[26606]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:20.337 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:20 smithi145 ceph-mon[21899]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:20.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:20 smithi172 ceph-mon[22747]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:22.024 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:10:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:10:21] "GET /metrics HTTP/1.1" 200 191495 "" "Prometheus/2.33.4" 2024-04-22T15:10:22.271 INFO:teuthology.orchestra.run.smithi145.stdout:55834574890 2024-04-22T15:10:22.271 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-22T15:10:22.294 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:22 smithi145 ceph-mon[26606]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:22.294 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:22 smithi145 ceph-mon[21899]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:22.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:22 smithi172 ceph-mon[22747]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:22.798 INFO:teuthology.orchestra.run.smithi145.stdout:133143986203 2024-04-22T15:10:22.798 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-22T15:10:24.317 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:24 smithi145 ceph-mon[26606]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:24.317 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:24 smithi145 ceph-mon[21899]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:24.341 INFO:teuthology.orchestra.run.smithi145.stdout:167503724566 2024-04-22T15:10:24.342 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-22T15:10:24.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:24 smithi172 ceph-mon[22747]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:25.531 INFO:teuthology.orchestra.run.smithi145.stdout:77309411365 2024-04-22T15:10:25.532 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-22T15:10:26.186 INFO:teuthology.orchestra.run.smithi145.stdout:111669149727 2024-04-22T15:10:26.187 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-22T15:10:26.340 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:26 smithi145 ceph-mon[26606]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:26.340 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:26 smithi145 ceph-mon[21899]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:26.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:26 smithi172 ceph-mon[22747]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:27.355 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:27 smithi145 ceph-mon[26606]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:27.356 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:27 smithi145 ceph-mon[21899]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:27.362 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T15:10:27.362 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T15:10:27.386 INFO:teuthology.orchestra.run.smithi145.stdout:201863462931 2024-04-22T15:10:27.386 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-22T15:10:27.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:27 smithi172 ceph-mon[22747]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:27.604 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T15:10:28.000 INFO:teuthology.orchestra.run.smithi145.stdout:236223201294 2024-04-22T15:10:28.001 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-22T15:10:28.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:10:27 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:10:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:10:30.285 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:30 smithi172 ceph-mon[22747]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:30.312 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T15:10:30.312 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T15:10:30.313 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T15:10:30.355 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:30 smithi145 ceph-mon[26606]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:30.355 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:30 smithi145 ceph-mon[21899]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:31.545 INFO:teuthology.orchestra.run.smithi145.stdout:34359738416 2024-04-22T15:10:31.545 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-22T15:10:31.927 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T15:10:32.176 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:32 smithi145 ceph-mon[26606]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:32.176 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:32 smithi145 ceph-mon[21899]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:32.177 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:10:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:10:31] "GET /metrics HTTP/1.1" 200 191492 "" "Prometheus/2.33.4" 2024-04-22T15:10:32.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:32 smithi172 ceph-mon[22747]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:34.283 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:34 smithi145 ceph-mon[21899]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:34.283 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:34 smithi145 ceph-mon[26606]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:34.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:34 smithi172 ceph-mon[22747]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:35.790 INFO:teuthology.orchestra.run.smithi145.stdout:133143986206 2024-04-22T15:10:36.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:36 smithi145 ceph-mon[21899]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:36.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:36 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2565645548' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-22T15:10:36.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:36 smithi145 ceph-mon[26606]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:36.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:36 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2565645548' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-22T15:10:36.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:36 smithi172 ceph-mon[22747]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:36.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:36 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2565645548' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-22T15:10:36.985 INFO:teuthology.orchestra.run.smithi145.stdout:201863462933 2024-04-22T15:10:37.203 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986203 got 133143986206 for osd.4 2024-04-22T15:10:37.203 DEBUG:teuthology.parallel:result is None 2024-04-22T15:10:37.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:37 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/1914395023' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-22T15:10:37.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:37 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/1914395023' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-22T15:10:37.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:37 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/1914395023' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-22T15:10:38.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:38 smithi145 ceph-mon[21899]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:38.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:38 smithi145 ceph-mon[26606]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:38.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:10:37 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:10:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:10:38.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:38 smithi172 ceph-mon[22747]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:38.699 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462931 got 201863462933 for osd.6 2024-04-22T15:10:38.700 DEBUG:teuthology.parallel:result is None 2024-04-22T15:10:40.079 INFO:teuthology.orchestra.run.smithi145.stdout:111669149731 2024-04-22T15:10:40.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:40 smithi145 ceph-mon[21899]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:40.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:40 smithi145 ceph-mon[26606]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:40.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:40 smithi172 ceph-mon[22747]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:40.785 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149727 got 111669149731 for osd.3 2024-04-22T15:10:40.785 DEBUG:teuthology.parallel:result is None 2024-04-22T15:10:41.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:41 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/1089612287' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-22T15:10:41.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:41 smithi172 ceph-mon[22747]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:41.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:41 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/1089612287' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-22T15:10:41.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:41 smithi145 ceph-mon[21899]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:41.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:41 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/1089612287' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-22T15:10:41.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:41 smithi145 ceph-mon[26606]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:42.045 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:10:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:10:41] "GET /metrics HTTP/1.1" 200 191492 "" "Prometheus/2.33.4" 2024-04-22T15:10:42.339 INFO:teuthology.orchestra.run.smithi145.stdout:55834574894 2024-04-22T15:10:42.528 INFO:teuthology.orchestra.run.smithi145.stdout:167503724570 2024-04-22T15:10:42.595 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:42 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2010192437' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-22T15:10:42.595 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:42 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2010192437' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-22T15:10:42.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:42 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2010192437' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-22T15:10:43.343 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574890 got 55834574894 for osd.1 2024-04-22T15:10:43.343 DEBUG:teuthology.parallel:result is None 2024-04-22T15:10:43.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:43 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/1914932039' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-22T15:10:43.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:43 smithi145 ceph-mon[21899]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:43.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:10:43.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:10:43.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:43 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/1914932039' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-22T15:10:43.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:43 smithi145 ceph-mon[26606]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:43.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:10:43.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:10:43.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:43 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/1914932039' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-22T15:10:43.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:43 smithi172 ceph-mon[22747]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:43.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:10:43.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:10:44.046 INFO:teuthology.orchestra.run.smithi145.stdout:236223201297 2024-04-22T15:10:44.253 INFO:teuthology.orchestra.run.smithi145.stdout:77309411369 2024-04-22T15:10:44.935 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411365 got 77309411369 for osd.2 2024-04-22T15:10:44.935 DEBUG:teuthology.parallel:result is None 2024-04-22T15:10:45.232 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:45 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/1976094266' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-22T15:10:45.232 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:45 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3419316113' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-22T15:10:45.233 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:45 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/1976094266' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-22T15:10:45.233 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:45 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3419316113' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-22T15:10:45.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:45 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/1976094266' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-22T15:10:45.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:45 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/3419316113' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-22T15:10:45.764 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201294 got 236223201297 for osd.7 2024-04-22T15:10:45.764 DEBUG:teuthology.parallel:result is None 2024-04-22T15:10:46.233 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T15:10:46.257 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724566 got 167503724570 for osd.5 2024-04-22T15:10:46.257 DEBUG:teuthology.parallel:result is None 2024-04-22T15:10:46.359 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:46 smithi145 ceph-mon[21899]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:46.360 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:46 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:46.360 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:46 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:46.360 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:46 smithi145 ceph-mon[26606]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:46.360 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:46 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:46.360 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:46 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:46.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:46 smithi172 ceph-mon[22747]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:46.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:46 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:46.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:46 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:47.351 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:47 smithi145 ceph-mon[26606]: Deploying daemon alertmanager.a on smithi145 2024-04-22T15:10:47.351 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:47 smithi145 ceph-mon[21899]: Deploying daemon alertmanager.a on smithi145 2024-04-22T15:10:47.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:47 smithi172 ceph-mon[22747]: Deploying daemon alertmanager.a on smithi145 2024-04-22T15:10:48.079 INFO:teuthology.orchestra.run.smithi145.stdout:34359738419 2024-04-22T15:10:48.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:48 smithi145 ceph-mon[21899]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:48.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:48 smithi145 ceph-mon[26606]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:48.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:10:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:10:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:10:48.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:48 smithi172 ceph-mon[22747]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:49.072 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738416 got 34359738419 for osd.0 2024-04-22T15:10:49.073 DEBUG:teuthology.parallel:result is None 2024-04-22T15:10:49.073 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-22T15:10:49.073 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-22T15:10:49.304 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T15:10:49.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:49 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3403789594' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-22T15:10:49.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:49 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3403789594' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-22T15:10:49.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:49 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/3403789594' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-22T15:10:50.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:50 smithi145 ceph-mon[21899]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:50.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:50 smithi145 ceph-mon[26606]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:50.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:50 smithi172 ceph-mon[22747]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:51.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:51 smithi145 ceph-mon[21899]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:51.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:51 smithi145 ceph-mon[26606]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:51.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:51 smithi172 ceph-mon[22747]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:52.007 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:10:51 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:10:51] "GET /metrics HTTP/1.1" 200 191490 "" "Prometheus/2.33.4" 2024-04-22T15:10:52.918 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-04-22T15:10:52.920 INFO:teuthology.orchestra.run.smithi145.stderr:dumped all 2024-04-22T15:10:54.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:54 smithi145 ceph-mon[21899]: from='client.24526 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T15:10:54.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:54 smithi145 ceph-mon[21899]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:54.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:54 smithi145 ceph-mon[26606]: from='client.24526 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T15:10:54.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:54 smithi145 ceph-mon[26606]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:54.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:54 smithi172 ceph-mon[22747]: from='client.24526 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T15:10:54.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:54 smithi172 ceph-mon[22747]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:55.067 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:10:54 smithi145 systemd[1]: Starting Ceph alertmanager.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:10:55.379 INFO:teuthology.orchestra.run.smithi145.stdout:{"pg_ready":true,"pg_map":{"version":37,"stamp":"2024-04-22T15:10:50.969000+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48956,"kb_used_data":6140,"kb_used_omap":0,"kb_used_meta":42752,"kb_avail":749912260,"statfs":{"total":767960285184,"available":767910154240,"internally_reserved":0,"allocated":6287360,"data_stored":3509919,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43778048},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002032"},"pg_stats":[{"pgid":"1.0","version":"60'128","reported_seq":266,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-04-22T15:09:43.040321+0000","last_change":"2024-04-22T15:09:27.010396+0000","last_active":"2024-04-22T15:09:43.040321+0000","last_peered":"2024-04-22T15:09:43.040321+0000","last_clean":"2024-04-22T15:09:43.040321+0000","last_became_active":"2024-04-22T15:09:27.009837+0000","last_became_peered":"2024-04-22T15:09:27.009837+0000","last_unstale":"2024-04-22T15:09:43.040321+0000","last_undegraded":"2024-04-22T15:09:43.040321+0000","last_fullsized":"2024-04-22T15:09:43.040321+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-22T15:07:30.220791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-22T15:07:30.220791+0000","last_clean_scrub_stamp":"2024-04-22T15:07:30.220791+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-23T23:27:42.210804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201299,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6092,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739060,"statfs":{"total":95995035648,"available":95988797440,"internally_reserved":0,"allocated":1118208,"data_stored":769265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 15:10:27 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.47599999999999998}]},{"osd":1,"last update":"Mon Apr 22 15:10:27 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.38800000000000001}]},{"osd":2,"last update":"Mon Apr 22 15:10:27 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.504},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.48699999999999999}]},{"osd":3,"last update":"Mon Apr 22 15:10:27 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.442}]},{"osd":4,"last update":"Mon Apr 22 15:10:27 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.41999999999999998}]},{"osd":5,"last update":"Mon Apr 22 15:10:27 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.0269999999999999,"5min":1.0269999999999999,"15min":1.0269999999999999},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.149,"5min":1.149,"15min":1.149},"last":0.51700000000000002}]},{"osd":6,"last update":"Mon Apr 22 15:10:27 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.0940000000000001,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.052,"5min":1.052,"15min":1.052},"last":0.54100000000000004}]}]},{"osd":6,"up_from":47,"seq":201863462936,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6156,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738996,"statfs":{"total":95995035648,"available":95988731904,"internally_reserved":0,"allocated":1118208,"data_stored":769265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 15:10:03 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.498}]},{"osd":1,"last update":"Mon Apr 22 15:10:03 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.36099999999999999}]},{"osd":2,"last update":"Mon Apr 22 15:10:03 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.38900000000000001}]},{"osd":3,"last update":"Mon Apr 22 15:10:03 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.45200000000000001}]},{"osd":4,"last update":"Mon Apr 22 15:10:03 2024","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.629},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.54500000000000004}]},{"osd":5,"last update":"Mon Apr 22 15:10:03 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.51800000000000002}]},{"osd":7,"last update":"Mon Apr 22 15:10:24 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.59199999999999997}]}]},{"osd":1,"up_from":13,"seq":55834574896,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6340,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738812,"statfs":{"total":95995035648,"available":95988543488,"internally_reserved":0,"allocated":585728,"data_stored":240395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 15:10:08 2024","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.34300000000000003,"15min":0.32700000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.54300000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"last":0.27600000000000002},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.35099999999999998,"15min":0.33900000000000002},"min":{"1min":0.254,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.56200000000000006,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.22500000000000001}]},{"osd":2,"last update":"Mon Apr 22 15:10:35 2024","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.36299999999999999,"15min":0.36599999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.48299999999999998,"5min":0.53600000000000003,"15min":0.53600000000000003},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.373,"5min":0.40300000000000002,"15min":0.40600000000000003},"min":{"1min":0.28000000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.57799999999999996,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.38500000000000001}]},{"osd":3,"last update":"Mon Apr 22 15:09:59 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.44900000000000001,"15min":0.45600000000000002},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.59399999999999997,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.42199999999999999,"15min":0.42599999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"last":0.39600000000000002}]},{"osd":4,"last update":"Mon Apr 22 15:10:19 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54700000000000004,"15min":0.54800000000000004},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.436},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.55600000000000005,"15min":0.55900000000000005},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.34000000000000002}]},{"osd":5,"last update":"Mon Apr 22 15:10:44 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.56599999999999995,"15min":0.56000000000000005},"min":{"1min":0.36799999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.58199999999999996,"15min":0.57799999999999996},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.40899999999999997}]},{"osd":6,"last update":"Mon Apr 22 15:10:03 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.379},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.49299999999999999}]},{"osd":7,"last update":"Mon Apr 22 15:10:23 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.45100000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738420,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6928,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738224,"statfs":{"total":95995035648,"available":95987941376,"internally_reserved":0,"allocated":1122304,"data_stored":769588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Apr 22 15:10:09 2024","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.35699999999999998,"15min":0.34399999999999997},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"last":0.255},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.36299999999999999,"15min":0.35399999999999998},"min":{"1min":0.27200000000000002,"5min":0.253,"15min":0.253},"max":{"1min":0.51300000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"last":0.308}]},{"osd":2,"last update":"Mon Apr 22 15:10:38 2024","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.437,"15min":0.44500000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.442,"15min":0.45300000000000001},"min":{"1min":0.25900000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.55000000000000004,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.32300000000000001}]},{"osd":3,"last update":"Mon Apr 22 15:09:58 2024","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.441,"15min":0.439},"min":{"1min":0.32300000000000001,"5min":0.317,"15min":0.317},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.433,"5min":0.433,"15min":0.433},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.34699999999999998}]},{"osd":4,"last update":"Mon Apr 22 15:10:20 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56499999999999995,"15min":0.56399999999999995},"min":{"1min":0.433,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.68899999999999995,"5min":0.745,"15min":0.745},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.53400000000000003,"15min":0.53200000000000003},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.51800000000000002}]},{"osd":5,"last update":"Mon Apr 22 15:10:38 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.40600000000000003,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.68700000000000006,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.55400000000000005,"15min":0.55100000000000005},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.76400000000000001,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.70699999999999996}]},{"osd":6,"last update":"Mon Apr 22 15:10:03 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.73299999999999998}]},{"osd":7,"last update":"Mon Apr 22 15:10:25 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.71399999999999997}]}]},{"osd":2,"up_from":18,"seq":77309411371,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6276,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738876,"statfs":{"total":95995035648,"available":95988609024,"internally_reserved":0,"allocated":585728,"data_stored":240337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 15:10:32 2024","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":0.35999999999999999,"15min":0.35499999999999998},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.44,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.24399999999999999},{"interface":"front","average":{"1min":0.376,"5min":0.375,"15min":0.36499999999999999},"min":{"1min":0.29299999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.53400000000000003,"5min":0.55900000000000005,"15min":0.55900000000000005},"last":0.26900000000000002}]},{"osd":1,"last update":"Mon Apr 22 15:10:32 2024","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":0.376,"15min":0.374},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.46899999999999997,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.38200000000000001,"5min":0.38800000000000001,"15min":0.38},"min":{"1min":0.27700000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.51700000000000002,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.28699999999999998}]},{"osd":3,"last update":"Mon Apr 22 15:09:56 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.26600000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.374},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.42499999999999999,"15min":0.42199999999999999},"min":{"1min":0.34000000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.55900000000000005,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.35199999999999998}]},{"osd":4,"last update":"Mon Apr 22 15:10:16 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":1.0509999999999999,"15min":1.133},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.876,"5min":14.195,"15min":14.195},"last":0.53800000000000003},{"interface":"front","average":{"1min":1.0760000000000001,"5min":1.163,"15min":1.177},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":7.8369999999999997,"5min":14.224,"15min":14.224},"last":0.629}]},{"osd":5,"last update":"Mon Apr 22 15:10:41 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":1.155,"15min":1.2509999999999999},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.77600000000000002,"5min":14.281000000000001,"15min":14.281000000000001},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.59599999999999997,"15min":0.59699999999999998},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.63700000000000001}]},{"osd":6,"last update":"Mon Apr 22 15:10:02 2024","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.61899999999999999}]},{"osd":7,"last update":"Mon Apr 22 15:10:24 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.53100000000000003}]}]},{"osd":3,"up_from":26,"seq":111669149734,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5700,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739452,"statfs":{"total":95995035648,"available":95989198848,"internally_reserved":0,"allocated":585728,"data_stored":240337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 15:09:54 2024","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.41699999999999998,"15min":0.42099999999999999},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.57399999999999995,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.44900000000000001,"15min":0.45400000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.25800000000000001}]},{"osd":1,"last update":"Mon Apr 22 15:09:54 2024","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.45000000000000001,"15min":0.45600000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.55300000000000005,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.26700000000000002},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.442,"15min":0.44900000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.29899999999999999}]},{"osd":2,"last update":"Mon Apr 22 15:09:54 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.45000000000000001,"15min":0.45300000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.622,"5min":0.627,"15min":0.627},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.48299999999999998,"15min":0.48899999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.314}]},{"osd":4,"last update":"Mon Apr 22 15:10:19 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.58399999999999996,"15min":0.57899999999999996},"min":{"1min":0.36799999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.59999999999999998,"15min":0.59899999999999998},"min":{"1min":0.36199999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.86499999999999999,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.35999999999999999}]},{"osd":5,"last update":"Mon Apr 22 15:10:42 2024","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.60799999999999998,"15min":0.59899999999999998},"min":{"1min":0.436,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.63900000000000001,"15min":0.63500000000000001},"min":{"1min":0.42399999999999999,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.90100000000000002,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.435}]},{"osd":6,"last update":"Mon Apr 22 15:10:01 2024","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.0069999999999999,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.44400000000000001}]},{"osd":7,"last update":"Mon Apr 22 15:10:28 2024","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.98299999999999998,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.47799999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986209,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5768,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739384,"statfs":{"total":95995035648,"available":95989129216,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 15:10:22 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.52500000000000002,"15min":0.52100000000000002},"min":{"1min":0.35899999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.53400000000000003,"15min":0.52800000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.79300000000000004,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.68700000000000006}]},{"osd":1,"last update":"Mon Apr 22 15:10:22 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.50700000000000001,"15min":0.503},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.71899999999999997,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.55400000000000005,"15min":0.55500000000000005},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.60599999999999998}]},{"osd":2,"last update":"Mon Apr 22 15:10:22 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.52800000000000002,"15min":0.52600000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.69399999999999995,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.53800000000000003,"15min":0.53200000000000003},"min":{"1min":0.377,"5min":0.373,"15min":0.373},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.71499999999999997}]},{"osd":3,"last update":"Mon Apr 22 15:10:22 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.55700000000000005,"15min":0.55100000000000005},"min":{"1min":0.45200000000000001,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.745,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.52900000000000003,"15min":0.52500000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.66500000000000004}]},{"osd":5,"last update":"Mon Apr 22 15:10:39 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.40300000000000002,"15min":0.39100000000000001},"min":{"1min":0.30299999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.65000000000000002,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.435,"15min":0.41899999999999998},"min":{"1min":0.26500000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.78400000000000003,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.48999999999999999}]},{"osd":6,"last update":"Mon Apr 22 15:10:02 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.46000000000000002}]},{"osd":7,"last update":"Mon Apr 22 15:10:28 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.54900000000000004}]}]},{"osd":5,"up_from":39,"seq":167503724572,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5696,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739456,"statfs":{"total":95995035648,"available":95989202944,"internally_reserved":0,"allocated":581632,"data_stored":240072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 15:10:40 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.52600000000000002,"15min":0.53000000000000003},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.755,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.76100000000000001},{"interface":"front","average":{"1min":0.495,"5min":0.499,"15min":0.499},"min":{"1min":0.312,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.64000000000000001,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.61899999999999999}]},{"osd":1,"last update":"Mon Apr 22 15:10:40 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.51300000000000001,"15min":0.51400000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.73599999999999999,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.628},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.52100000000000002,"15min":0.52600000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.50900000000000001}]},{"osd":2,"last update":"Mon Apr 22 15:10:40 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.52100000000000002,"15min":0.52500000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.52800000000000002,"15min":0.53200000000000003},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.71299999999999997,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.54800000000000004}]},{"osd":3,"last update":"Mon Apr 22 15:10:40 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.53700000000000003,"15min":0.54200000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.69499999999999995,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.79000000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.67400000000000004}]},{"osd":4,"last update":"Mon Apr 22 15:10:40 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.40200000000000002,"15min":0.39200000000000002},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.41199999999999998,"15min":0.39900000000000002},"min":{"1min":0.29499999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.71399999999999997}]},{"osd":6,"last update":"Mon Apr 22 15:10:01 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.83299999999999996},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.79200000000000004}]},{"osd":7,"last update":"Mon Apr 22 15:10:26 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.80100000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-22T15:10:55.381 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-22T15:10:55.623 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:10:55 smithi145 podman[44915]: 2024-04-22 15:10:55.466835523 +0000 UTC m=+0.024802294 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-04-22T15:10:55.623 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:10:55 smithi145 podman[44915]: 2024-04-22 15:10:55.586445847 +0000 UTC m=+0.144412644 volume create e3f30f112e296b24915b9316c00697391643fdde220f7775c054becbc0af3b71 2024-04-22T15:10:56.008 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:10:55 smithi145 podman[44915]: 2024-04-22 15:10:55.695301094 +0000 UTC m=+0.253267846 container create 61c101b04b81af627f21b73bbd83549be8027b44257db7909141c4105de5836b (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T15:10:56.127 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T15:10:56.259 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:56 smithi145 ceph-mon[21899]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:56.260 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:56 smithi145 ceph-mon[26606]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:56.260 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:10:56 smithi145 podman[44915]: 2024-04-22 15:10:56.006185239 +0000 UTC m=+0.564152010 container init 61c101b04b81af627f21b73bbd83549be8027b44257db7909141c4105de5836b (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T15:10:56.260 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:10:56 smithi145 podman[44915]: 2024-04-22 15:10:56.009534819 +0000 UTC m=+0.567501572 container start 61c101b04b81af627f21b73bbd83549be8027b44257db7909141c4105de5836b (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T15:10:56.260 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:10:56 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[44942]: level=info ts=2024-04-22T15:10:56.020Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-04-22T15:10:56.260 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:10:56 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[44942]: level=info ts=2024-04-22T15:10:56.020Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-04-22T15:10:56.260 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:10:56 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[44942]: level=info ts=2024-04-22T15:10:56.021Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.145 port=9094 2024-04-22T15:10:56.261 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:10:56 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[44942]: level=info ts=2024-04-22T15:10:56.022Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-22T15:10:56.261 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:10:56 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[44942]: level=info ts=2024-04-22T15:10:56.048Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-22T15:10:56.261 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:10:56 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[44942]: level=info ts=2024-04-22T15:10:56.048Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-22T15:10:56.261 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:10:56 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[44942]: level=info ts=2024-04-22T15:10:56.050Z caller=main.go:518 msg=Listening address=:9093 2024-04-22T15:10:56.261 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:10:56 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[44942]: level=info ts=2024-04-22T15:10:56.051Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-04-22T15:10:56.261 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:10:56 smithi145 bash[44915]: 61c101b04b81af627f21b73bbd83549be8027b44257db7909141c4105de5836b 2024-04-22T15:10:56.261 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:10:56 smithi145 systemd[1]: Started Ceph alertmanager.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:10:56.284 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:56 smithi172 ceph-mon[22747]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:57.273 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:57 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:57.274 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:57 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:57.274 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:57 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:57.274 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:57 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T15:10:57.274 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:57 smithi145 ceph-mon[26606]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T15:10:57.274 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:57 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:57.274 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:57 smithi145 ceph-mon[26606]: Deploying daemon grafana.a on smithi172 2024-04-22T15:10:57.274 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:57 smithi145 ceph-mon[26606]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:57.274 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:57 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:57.274 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:57 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:57.275 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:57 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:57.275 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:57 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T15:10:57.275 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:57 smithi145 ceph-mon[21899]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T15:10:57.275 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:57 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:57.275 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:57 smithi145 ceph-mon[21899]: Deploying daemon grafana.a on smithi172 2024-04-22T15:10:57.275 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:57 smithi145 ceph-mon[21899]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:57.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:57 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:57.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:57 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:57.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:57 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:57.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:57 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T15:10:57.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:57 smithi172 ceph-mon[22747]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T15:10:57.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:57 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:57.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:57 smithi172 ceph-mon[22747]: Deploying daemon grafana.a on smithi172 2024-04-22T15:10:57.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:57 smithi172 ceph-mon[22747]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:10:57.806 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-04-22T15:10:57.807 INFO:teuthology.orchestra.run.smithi145.stderr:dumped all 2024-04-22T15:10:58.122 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:10:58 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[44942]: level=info ts=2024-04-22T15:10:58.023Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000778566s 2024-04-22T15:10:58.421 INFO:teuthology.orchestra.run.smithi145.stdout:{"pg_ready":true,"pg_map":{"version":40,"stamp":"2024-04-22T15:10:56.970106+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48956,"kb_used_data":6140,"kb_used_omap":0,"kb_used_meta":42752,"kb_avail":749912260,"statfs":{"total":767960285184,"available":767910154240,"internally_reserved":0,"allocated":6287360,"data_stored":3509919,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43778048},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002162"},"pg_stats":[{"pgid":"1.0","version":"60'128","reported_seq":266,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-04-22T15:09:43.040321+0000","last_change":"2024-04-22T15:09:27.010396+0000","last_active":"2024-04-22T15:09:43.040321+0000","last_peered":"2024-04-22T15:09:43.040321+0000","last_clean":"2024-04-22T15:09:43.040321+0000","last_became_active":"2024-04-22T15:09:27.009837+0000","last_became_peered":"2024-04-22T15:09:27.009837+0000","last_unstale":"2024-04-22T15:09:43.040321+0000","last_undegraded":"2024-04-22T15:09:43.040321+0000","last_fullsized":"2024-04-22T15:09:43.040321+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-22T15:07:30.220791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-22T15:07:30.220791+0000","last_clean_scrub_stamp":"2024-04-22T15:07:30.220791+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-23T23:27:42.210804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201300,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6092,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739060,"statfs":{"total":95995035648,"available":95988797440,"internally_reserved":0,"allocated":1118208,"data_stored":769265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 15:10:27 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.56999999999999995}]},{"osd":1,"last update":"Mon Apr 22 15:10:27 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.60099999999999998}]},{"osd":2,"last update":"Mon Apr 22 15:10:27 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.497},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.63200000000000001}]},{"osd":3,"last update":"Mon Apr 22 15:10:27 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.51200000000000001}]},{"osd":4,"last update":"Mon Apr 22 15:10:27 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.53900000000000003}]},{"osd":5,"last update":"Mon Apr 22 15:10:27 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.0269999999999999,"5min":1.0269999999999999,"15min":1.0269999999999999},"last":0.434},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.149,"5min":1.149,"15min":1.149},"last":0.52700000000000002}]},{"osd":6,"last update":"Mon Apr 22 15:10:27 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.0940000000000001,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.052,"5min":1.052,"15min":1.052},"last":0.65700000000000003}]}]},{"osd":6,"up_from":47,"seq":201863462937,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6156,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738996,"statfs":{"total":95995035648,"available":95988731904,"internally_reserved":0,"allocated":1118208,"data_stored":769265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 15:10:03 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.498}]},{"osd":1,"last update":"Mon Apr 22 15:10:03 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.36099999999999999}]},{"osd":2,"last update":"Mon Apr 22 15:10:03 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.38900000000000001}]},{"osd":3,"last update":"Mon Apr 22 15:10:03 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.45200000000000001}]},{"osd":4,"last update":"Mon Apr 22 15:10:03 2024","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.629},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.54500000000000004}]},{"osd":5,"last update":"Mon Apr 22 15:10:03 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.51800000000000002}]},{"osd":7,"last update":"Mon Apr 22 15:10:24 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.59199999999999997}]}]},{"osd":1,"up_from":13,"seq":55834574897,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6340,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738812,"statfs":{"total":95995035648,"available":95988543488,"internally_reserved":0,"allocated":585728,"data_stored":240395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 15:10:08 2024","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.34300000000000003,"15min":0.32700000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.54300000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.35099999999999998,"15min":0.33900000000000002},"min":{"1min":0.254,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.56200000000000006,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.308}]},{"osd":2,"last update":"Mon Apr 22 15:10:35 2024","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.36299999999999999,"15min":0.36599999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.48299999999999998,"5min":0.53600000000000003,"15min":0.53600000000000003},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.373,"5min":0.40300000000000002,"15min":0.40600000000000003},"min":{"1min":0.28000000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.57799999999999996,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.33400000000000002}]},{"osd":3,"last update":"Mon Apr 22 15:09:59 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.44900000000000001,"15min":0.45600000000000002},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.59399999999999997,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.42199999999999999,"15min":0.42599999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"last":0.41499999999999998}]},{"osd":4,"last update":"Mon Apr 22 15:10:19 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54700000000000004,"15min":0.54800000000000004},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.55600000000000005,"15min":0.55900000000000005},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.72599999999999998}]},{"osd":5,"last update":"Mon Apr 22 15:10:44 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.56599999999999995,"15min":0.56000000000000005},"min":{"1min":0.36799999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.83199999999999996},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.58199999999999996,"15min":0.57799999999999996},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.81299999999999994}]},{"osd":6,"last update":"Mon Apr 22 15:10:03 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.66000000000000003}]},{"osd":7,"last update":"Mon Apr 22 15:10:23 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.73899999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738421,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6928,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738224,"statfs":{"total":95995035648,"available":95987941376,"internally_reserved":0,"allocated":1122304,"data_stored":769588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Apr 22 15:10:09 2024","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.35699999999999998,"15min":0.34399999999999997},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"last":0.246},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.36299999999999999,"15min":0.35399999999999998},"min":{"1min":0.27200000000000002,"5min":0.253,"15min":0.253},"max":{"1min":0.51300000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"last":0.318}]},{"osd":2,"last update":"Mon Apr 22 15:10:38 2024","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.437,"15min":0.44500000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.442,"15min":0.45300000000000001},"min":{"1min":0.25900000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.55000000000000004,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.29199999999999998}]},{"osd":3,"last update":"Mon Apr 22 15:09:58 2024","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.441,"15min":0.439},"min":{"1min":0.32300000000000001,"5min":0.317,"15min":0.317},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.433,"5min":0.433,"15min":0.433},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.26100000000000001}]},{"osd":4,"last update":"Mon Apr 22 15:10:20 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56499999999999995,"15min":0.56399999999999995},"min":{"1min":0.433,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.68899999999999995,"5min":0.745,"15min":0.745},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.53400000000000003,"15min":0.53200000000000003},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.40000000000000002}]},{"osd":5,"last update":"Mon Apr 22 15:10:38 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.40600000000000003,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.68700000000000006,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.55400000000000005,"15min":0.55100000000000005},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.76400000000000001,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.51100000000000001}]},{"osd":6,"last update":"Mon Apr 22 15:10:03 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.443}]},{"osd":7,"last update":"Mon Apr 22 15:10:25 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.373},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.45300000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411372,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6276,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738876,"statfs":{"total":95995035648,"available":95988609024,"internally_reserved":0,"allocated":585728,"data_stored":240337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 15:10:32 2024","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":0.35999999999999999,"15min":0.35499999999999998},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.44,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.376,"5min":0.375,"15min":0.36499999999999999},"min":{"1min":0.29299999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.53400000000000003,"5min":0.55900000000000005,"15min":0.55900000000000005},"last":0.40600000000000003}]},{"osd":1,"last update":"Mon Apr 22 15:10:32 2024","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":0.376,"15min":0.374},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.46899999999999997,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.44},{"interface":"front","average":{"1min":0.38200000000000001,"5min":0.38800000000000001,"15min":0.38},"min":{"1min":0.27700000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.51700000000000002,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.35799999999999998}]},{"osd":3,"last update":"Mon Apr 22 15:09:56 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.26600000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.309},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.42499999999999999,"15min":0.42199999999999999},"min":{"1min":0.34000000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.55900000000000005,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.38800000000000001}]},{"osd":4,"last update":"Mon Apr 22 15:10:16 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":1.0509999999999999,"15min":1.133},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.876,"5min":14.195,"15min":14.195},"last":0.501},{"interface":"front","average":{"1min":1.0760000000000001,"5min":1.163,"15min":1.177},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":7.8369999999999997,"5min":14.224,"15min":14.224},"last":0.58499999999999996}]},{"osd":5,"last update":"Mon Apr 22 15:10:41 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":1.155,"15min":1.2509999999999999},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.77600000000000002,"5min":14.281000000000001,"15min":14.281000000000001},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.59599999999999997,"15min":0.59699999999999998},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.52400000000000002}]},{"osd":6,"last update":"Mon Apr 22 15:10:02 2024","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.59199999999999997}]},{"osd":7,"last update":"Mon Apr 22 15:10:24 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.60999999999999999}]}]},{"osd":3,"up_from":26,"seq":111669149735,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5700,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739452,"statfs":{"total":95995035648,"available":95989198848,"internally_reserved":0,"allocated":585728,"data_stored":240337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 15:09:54 2024","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.41699999999999998,"15min":0.42099999999999999},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.57399999999999995,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.31},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.44900000000000001,"15min":0.45400000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.45000000000000001}]},{"osd":1,"last update":"Mon Apr 22 15:09:54 2024","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.45000000000000001,"15min":0.45600000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.55300000000000005,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.442,"15min":0.44900000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.39600000000000002}]},{"osd":2,"last update":"Mon Apr 22 15:09:54 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.45000000000000001,"15min":0.45300000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.622,"5min":0.627,"15min":0.627},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.48299999999999998,"15min":0.48899999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.41499999999999998}]},{"osd":4,"last update":"Mon Apr 22 15:10:19 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.58399999999999996,"15min":0.57899999999999996},"min":{"1min":0.36799999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.59999999999999998,"15min":0.59899999999999998},"min":{"1min":0.36199999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.86499999999999999,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.54700000000000004}]},{"osd":5,"last update":"Mon Apr 22 15:10:42 2024","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.60799999999999998,"15min":0.59899999999999998},"min":{"1min":0.436,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.63900000000000001,"15min":0.63500000000000001},"min":{"1min":0.42399999999999999,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.90100000000000002,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.503}]},{"osd":6,"last update":"Mon Apr 22 15:10:01 2024","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.0069999999999999,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.60999999999999999}]},{"osd":7,"last update":"Mon Apr 22 15:10:28 2024","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.98299999999999998,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.64000000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986210,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5768,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739384,"statfs":{"total":95995035648,"available":95989129216,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 15:10:22 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.52500000000000002,"15min":0.52100000000000002},"min":{"1min":0.35899999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.53400000000000003,"15min":0.52800000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.79300000000000004,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.67200000000000004}]},{"osd":1,"last update":"Mon Apr 22 15:10:22 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.50700000000000001,"15min":0.503},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.71899999999999997,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.55400000000000005,"15min":0.55500000000000005},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.71199999999999997}]},{"osd":2,"last update":"Mon Apr 22 15:10:22 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.52800000000000002,"15min":0.52600000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.69399999999999995,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.53800000000000003,"15min":0.53200000000000003},"min":{"1min":0.377,"5min":0.373,"15min":0.373},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.628}]},{"osd":3,"last update":"Mon Apr 22 15:10:22 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.55700000000000005,"15min":0.55100000000000005},"min":{"1min":0.45200000000000001,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.745,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.52900000000000003,"15min":0.52500000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.69099999999999995}]},{"osd":5,"last update":"Mon Apr 22 15:10:39 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.40300000000000002,"15min":0.39100000000000001},"min":{"1min":0.30299999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.65000000000000002,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.435,"15min":0.41899999999999998},"min":{"1min":0.26500000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.78400000000000003,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.82799999999999996}]},{"osd":6,"last update":"Mon Apr 22 15:10:02 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.77300000000000002},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.76300000000000001}]},{"osd":7,"last update":"Mon Apr 22 15:10:28 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.80700000000000005},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.751}]}]},{"osd":5,"up_from":39,"seq":167503724573,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5696,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739456,"statfs":{"total":95995035648,"available":95989202944,"internally_reserved":0,"allocated":581632,"data_stored":240072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 15:10:40 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.52600000000000002,"15min":0.53000000000000003},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.755,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.495,"5min":0.499,"15min":0.499},"min":{"1min":0.312,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.64000000000000001,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.65800000000000003}]},{"osd":1,"last update":"Mon Apr 22 15:10:40 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.51300000000000001,"15min":0.51400000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.73599999999999999,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.52100000000000002,"15min":0.52600000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.53300000000000003}]},{"osd":2,"last update":"Mon Apr 22 15:10:40 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.52100000000000002,"15min":0.52500000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.52800000000000002,"15min":0.53200000000000003},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.71299999999999997,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.54600000000000004}]},{"osd":3,"last update":"Mon Apr 22 15:10:40 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.53700000000000003,"15min":0.54200000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.69499999999999995,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.79000000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.628}]},{"osd":4,"last update":"Mon Apr 22 15:10:40 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.40200000000000002,"15min":0.39200000000000002},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.437},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.41199999999999998,"15min":0.39900000000000002},"min":{"1min":0.29499999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.502}]},{"osd":6,"last update":"Mon Apr 22 15:10:01 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.57499999999999996}]},{"osd":7,"last update":"Mon Apr 22 15:10:26 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.67500000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-22T15:10:58.423 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-22T15:10:58.423 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-22T15:10:58.423 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-22T15:10:58.423 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-22T15:10:58.467 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:10:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:10:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:10:58.634 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T15:10:59.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:59 smithi145 ceph-mon[26606]: from='client.24532 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T15:10:59.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:10:59 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:59.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:59 smithi145 ceph-mon[21899]: from='client.24532 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T15:10:59.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:10:59 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:10:59.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:59 smithi172 ceph-mon[22747]: from='client.24532 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T15:10:59.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:10:59 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:00.368 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:00 smithi172 ceph-mon[22747]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:00.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:00 smithi145 ceph-mon[26606]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:00.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:00 smithi145 ceph-mon[21899]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:00.391 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-04-22T15:11:00.391 INFO:teuthology.orchestra.run.smithi145.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-22T15:11:01.068 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-22T15:11:01.068 INFO:tasks.cephadm:Setup complete, yielding 2024-04-22T15:11:01.069 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-22T15:11:01.078 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi145.front.sepia.ceph.com 2024-04-22T15:11:01.079 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'radosgw-admin realm create --rgw-realm=r --default' 2024-04-22T15:11:01.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:01 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/725662058' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-22T15:11:01.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:01 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/725662058' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-22T15:11:01.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:01 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/725662058' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-22T15:11:02.029 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:11:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:11:01] "GET /metrics HTTP/1.1" 200 191494 "" "Prometheus/2.33.4" 2024-04-22T15:11:02.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:02 smithi145 ceph-mon[21899]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:02.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:02 smithi145 ceph-mon[26606]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:02.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:02 smithi172 ceph-mon[22747]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:03.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:03 smithi172 ceph-mon[22747]: pgmap v43: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:03.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:03 smithi145 ceph-mon[21899]: pgmap v43: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:03.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:03 smithi145 ceph-mon[26606]: pgmap v43: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:04.158 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:11:04.158 INFO:teuthology.orchestra.run.smithi145.stdout: "id": "e694c198-925d-4328-a804-67b7d03db749", 2024-04-22T15:11:04.158 INFO:teuthology.orchestra.run.smithi145.stdout: "name": "r", 2024-04-22T15:11:04.158 INFO:teuthology.orchestra.run.smithi145.stdout: "current_period": "9716de0c-e911-4ff5-b420-06054021de9e", 2024-04-22T15:11:04.158 INFO:teuthology.orchestra.run.smithi145.stdout: "epoch": 1 2024-04-22T15:11:04.158 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:11:04.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:04 smithi172 ceph-mon[22747]: osdmap e61: 8 total, 8 up, 8 in 2024-04-22T15:11:04.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:04 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2821413305' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-22T15:11:04.473 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:04 smithi145 ceph-mon[26606]: osdmap e61: 8 total, 8 up, 8 in 2024-04-22T15:11:04.473 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:04 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2821413305' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-22T15:11:04.473 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:04 smithi145 ceph-mon[21899]: osdmap e61: 8 total, 8 up, 8 in 2024-04-22T15:11:04.473 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:04 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2821413305' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-22T15:11:04.768 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'radosgw-admin zonegroup create --rgw-zonegroup=default --master --default' 2024-04-22T15:11:05.360 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:05 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2821413305' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-04-22T15:11:05.360 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:05 smithi145 ceph-mon[26606]: osdmap e62: 8 total, 8 up, 8 in 2024-04-22T15:11:05.360 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:05 smithi145 ceph-mon[26606]: pgmap v46: 33 pgs: 32 creating+peering, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:05.360 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:05 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2821413305' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-04-22T15:11:05.360 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:05 smithi145 ceph-mon[21899]: osdmap e62: 8 total, 8 up, 8 in 2024-04-22T15:11:05.360 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:05 smithi145 ceph-mon[21899]: pgmap v46: 33 pgs: 32 creating+peering, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:05.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:05 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2821413305' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-04-22T15:11:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:05 smithi172 ceph-mon[22747]: osdmap e62: 8 total, 8 up, 8 in 2024-04-22T15:11:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:05 smithi172 ceph-mon[22747]: pgmap v46: 33 pgs: 32 creating+peering, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:06.336 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:06 smithi145 ceph-mon[21899]: osdmap e63: 8 total, 8 up, 8 in 2024-04-22T15:11:06.337 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:11:06 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[44942]: level=info ts=2024-04-22T15:11:06.026Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003885646s 2024-04-22T15:11:06.337 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:06 smithi145 ceph-mon[26606]: osdmap e63: 8 total, 8 up, 8 in 2024-04-22T15:11:06.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:06 smithi172 ceph-mon[22747]: osdmap e63: 8 total, 8 up, 8 in 2024-04-22T15:11:06.508 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:11:06.508 INFO:teuthology.orchestra.run.smithi145.stdout: "id": "f46bedf2-002a-4b17-8d46-37ed97c44cee", 2024-04-22T15:11:06.508 INFO:teuthology.orchestra.run.smithi145.stdout: "name": "default", 2024-04-22T15:11:06.508 INFO:teuthology.orchestra.run.smithi145.stdout: "api_name": "default", 2024-04-22T15:11:06.508 INFO:teuthology.orchestra.run.smithi145.stdout: "is_master": "true", 2024-04-22T15:11:06.508 INFO:teuthology.orchestra.run.smithi145.stdout: "endpoints": [], 2024-04-22T15:11:06.508 INFO:teuthology.orchestra.run.smithi145.stdout: "hostnames": [], 2024-04-22T15:11:06.508 INFO:teuthology.orchestra.run.smithi145.stdout: "hostnames_s3website": [], 2024-04-22T15:11:06.508 INFO:teuthology.orchestra.run.smithi145.stdout: "master_zone": "", 2024-04-22T15:11:06.508 INFO:teuthology.orchestra.run.smithi145.stdout: "zones": [], 2024-04-22T15:11:06.509 INFO:teuthology.orchestra.run.smithi145.stdout: "placement_targets": [], 2024-04-22T15:11:06.509 INFO:teuthology.orchestra.run.smithi145.stdout: "default_placement": "", 2024-04-22T15:11:06.509 INFO:teuthology.orchestra.run.smithi145.stdout: "realm_id": "e694c198-925d-4328-a804-67b7d03db749", 2024-04-22T15:11:06.509 INFO:teuthology.orchestra.run.smithi145.stdout: "sync_policy": { 2024-04-22T15:11:06.509 INFO:teuthology.orchestra.run.smithi145.stdout: "groups": [] 2024-04-22T15:11:06.509 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:11:06.509 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:11:07.137 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default' 2024-04-22T15:11:07.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:07 smithi145 ceph-mon[26606]: pgmap v48: 33 pgs: 32 creating+peering, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:07.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:07 smithi145 ceph-mon[21899]: pgmap v48: 33 pgs: 32 creating+peering, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:07.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:07 smithi172 ceph-mon[22747]: pgmap v48: 33 pgs: 32 creating+peering, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:08.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:11:07 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:11:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:11:08.912 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:11:08.912 INFO:teuthology.orchestra.run.smithi145.stdout: "id": "f1401a66-b907-4f13-b8a7-1a3c48a44285", 2024-04-22T15:11:08.912 INFO:teuthology.orchestra.run.smithi145.stdout: "name": "z", 2024-04-22T15:11:08.912 INFO:teuthology.orchestra.run.smithi145.stdout: "domain_root": "z.rgw.meta:root", 2024-04-22T15:11:08.912 INFO:teuthology.orchestra.run.smithi145.stdout: "control_pool": "z.rgw.control", 2024-04-22T15:11:08.912 INFO:teuthology.orchestra.run.smithi145.stdout: "gc_pool": "z.rgw.log:gc", 2024-04-22T15:11:08.912 INFO:teuthology.orchestra.run.smithi145.stdout: "lc_pool": "z.rgw.log:lc", 2024-04-22T15:11:08.912 INFO:teuthology.orchestra.run.smithi145.stdout: "log_pool": "z.rgw.log", 2024-04-22T15:11:08.913 INFO:teuthology.orchestra.run.smithi145.stdout: "intent_log_pool": "z.rgw.log:intent", 2024-04-22T15:11:08.913 INFO:teuthology.orchestra.run.smithi145.stdout: "usage_log_pool": "z.rgw.log:usage", 2024-04-22T15:11:08.913 INFO:teuthology.orchestra.run.smithi145.stdout: "roles_pool": "z.rgw.meta:roles", 2024-04-22T15:11:08.913 INFO:teuthology.orchestra.run.smithi145.stdout: "reshard_pool": "z.rgw.log:reshard", 2024-04-22T15:11:08.913 INFO:teuthology.orchestra.run.smithi145.stdout: "user_keys_pool": "z.rgw.meta:users.keys", 2024-04-22T15:11:08.913 INFO:teuthology.orchestra.run.smithi145.stdout: "user_email_pool": "z.rgw.meta:users.email", 2024-04-22T15:11:08.913 INFO:teuthology.orchestra.run.smithi145.stdout: "user_swift_pool": "z.rgw.meta:users.swift", 2024-04-22T15:11:08.913 INFO:teuthology.orchestra.run.smithi145.stdout: "user_uid_pool": "z.rgw.meta:users.uid", 2024-04-22T15:11:08.913 INFO:teuthology.orchestra.run.smithi145.stdout: "otp_pool": "z.rgw.otp", 2024-04-22T15:11:08.913 INFO:teuthology.orchestra.run.smithi145.stdout: "system_key": { 2024-04-22T15:11:08.913 INFO:teuthology.orchestra.run.smithi145.stdout: "access_key": "", 2024-04-22T15:11:08.913 INFO:teuthology.orchestra.run.smithi145.stdout: "secret_key": "" 2024-04-22T15:11:08.913 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:11:08.913 INFO:teuthology.orchestra.run.smithi145.stdout: "placement_pools": [ 2024-04-22T15:11:08.914 INFO:teuthology.orchestra.run.smithi145.stdout: { 2024-04-22T15:11:08.914 INFO:teuthology.orchestra.run.smithi145.stdout: "key": "default-placement", 2024-04-22T15:11:08.914 INFO:teuthology.orchestra.run.smithi145.stdout: "val": { 2024-04-22T15:11:08.914 INFO:teuthology.orchestra.run.smithi145.stdout: "index_pool": "z.rgw.buckets.index", 2024-04-22T15:11:08.914 INFO:teuthology.orchestra.run.smithi145.stdout: "storage_classes": { 2024-04-22T15:11:08.914 INFO:teuthology.orchestra.run.smithi145.stdout: "STANDARD": { 2024-04-22T15:11:08.914 INFO:teuthology.orchestra.run.smithi145.stdout: "data_pool": "z.rgw.buckets.data" 2024-04-22T15:11:08.914 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:11:08.914 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:11:08.914 INFO:teuthology.orchestra.run.smithi145.stdout: "data_extra_pool": "z.rgw.buckets.non-ec", 2024-04-22T15:11:08.914 INFO:teuthology.orchestra.run.smithi145.stdout: "index_type": 0 2024-04-22T15:11:08.914 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:11:08.915 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:11:08.915 INFO:teuthology.orchestra.run.smithi145.stdout: ], 2024-04-22T15:11:08.915 INFO:teuthology.orchestra.run.smithi145.stdout: "realm_id": "e694c198-925d-4328-a804-67b7d03db749", 2024-04-22T15:11:08.915 INFO:teuthology.orchestra.run.smithi145.stdout: "notif_pool": "z.rgw.log:notif" 2024-04-22T15:11:08.915 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:11:09.663 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'radosgw-admin period update --rgw-realm=r --commit' 2024-04-22T15:11:10.270 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:10 smithi145 ceph-mon[26606]: pgmap v49: 33 pgs: 32 creating+peering, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:10.270 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:10 smithi145 ceph-mon[21899]: pgmap v49: 33 pgs: 32 creating+peering, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:10.454 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:10 smithi172 ceph-mon[22747]: pgmap v49: 33 pgs: 32 creating+peering, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:12.353 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-mon[22747]: pgmap v50: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 912 B/s rd, 2.0 KiB/s wr, 3 op/s 2024-04-22T15:11:12.353 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 systemd[1]: Starting Ceph grafana.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:11:12.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:12 smithi145 ceph-mon[21899]: pgmap v50: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 912 B/s rd, 2.0 KiB/s wr, 3 op/s 2024-04-22T15:11:12.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:12 smithi145 ceph-mon[26606]: pgmap v50: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 912 B/s rd, 2.0 KiB/s wr, 3 op/s 2024-04-22T15:11:12.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:11:11 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:11:11] "GET /metrics HTTP/1.1" 200 191494 "" "Prometheus/2.33.4" 2024-04-22T15:11:12.609 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 podman[38858]: 2024-04-22 15:11:12.25267459 +0000 UTC m=+0.017372977 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-04-22T15:11:12.610 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 podman[38858]: 2024-04-22 15:11:12.362751177 +0000 UTC m=+0.127449553 container create b5aa7bda58720a225de10612e27bfb4c9de1c637292ea33d5545a483eeb764ba (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, architecture=x86_64, distribution-scope=public, vendor=Red Hat, Inc., maintainer=Paul Cuzner , vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-03-28T10:36:18.413762, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, com.redhat.component=ubi8-container, io.buildah.version=1.24.2, release=236.1648460182, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, description=Ceph Grafana Container) 2024-04-22T15:11:12.610 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 podman[38858]: 2024-04-22 15:11:12.607238792 +0000 UTC m=+0.371937170 container init b5aa7bda58720a225de10612e27bfb4c9de1c637292ea33d5545a483eeb764ba (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, architecture=x86_64, io.openshift.tags=base rhel8, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.24.2, vendor=Red Hat, Inc., name=ubi8, release=236.1648460182, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, build-date=2022-03-28T10:36:18.413762, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-04-22T15:11:12.862 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 podman[38858]: 2024-04-22 15:11:12.609956998 +0000 UTC m=+0.374655378 container start b5aa7bda58720a225de10612e27bfb4c9de1c637292ea33d5545a483eeb764ba (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a, vendor=Red Hat, Inc., version=8.5, architecture=x86_64, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, com.redhat.component=ubi8-container, distribution-scope=public, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, release=236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, build-date=2022-03-28T10:36:18.413762, io.openshift.expose-services=, io.buildah.version=1.24.2, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-04-22T15:11:12.862 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-04-22T15:11:12.863 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-04-22T15:11:12.863 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-04-22T15:11:12.863 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-04-22T15:11:12.863 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-04-22T15:11:12.863 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-04-22T15:11:12.863 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-04-22T15:11:12.863 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-04-22T15:11:12.863 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-04-22T15:11:12.863 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-04-22T15:11:12.863 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-04-22T15:11:12.863 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-04-22T15:11:12.864 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-04-22T15:11:12.864 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="App mode production" logger=settings 2024-04-22T15:11:12.864 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-04-22T15:11:12.864 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-04-22T15:11:12.864 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-04-22T15:11:12.865 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2024-04-22T15:11:12.865 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2024-04-22T15:11:12.865 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2024-04-22T15:11:12.865 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2024-04-22T15:11:12.865 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2024-04-22T15:11:12.865 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2024-04-22T15:11:12.865 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2024-04-22T15:11:12.865 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2024-04-22T15:11:12.865 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2024-04-22T15:11:12.865 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2024-04-22T15:11:12.865 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-04-22T15:11:12.865 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2024-04-22T15:11:12.865 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2024-04-22T15:11:12.866 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2024-04-22T15:11:12.866 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2024-04-22T15:11:12.866 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2024-04-22T15:11:12.866 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2024-04-22T15:11:12.866 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2024-04-22T15:11:12.866 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2024-04-22T15:11:12.866 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2024-04-22T15:11:12.866 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2024-04-22T15:11:12.866 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2024-04-22T15:11:12.866 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2024-04-22T15:11:12.866 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2024-04-22T15:11:12.866 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2024-04-22T15:11:12.866 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2024-04-22T15:11:12.867 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2024-04-22T15:11:12.867 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2024-04-22T15:11:12.867 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2024-04-22T15:11:12.867 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-04-22T15:11:12.867 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2024-04-22T15:11:12.867 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2024-04-22T15:11:12.867 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2024-04-22T15:11:12.867 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2024-04-22T15:11:12.867 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2024-04-22T15:11:12.867 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2024-04-22T15:11:12.868 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2024-04-22T15:11:12.868 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2024-04-22T15:11:12.868 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2024-04-22T15:11:12.868 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2024-04-22T15:11:12.869 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2024-04-22T15:11:12.869 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2024-04-22T15:11:12.869 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2024-04-22T15:11:12.869 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2024-04-22T15:11:12.869 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2024-04-22T15:11:12.869 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2024-04-22T15:11:12.869 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 bash[38858]: b5aa7bda58720a225de10612e27bfb4c9de1c637292ea33d5545a483eeb764ba 2024-04-22T15:11:12.869 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2024-04-22T15:11:12.869 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2024-04-22T15:11:12.869 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2024-04-22T15:11:12.869 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 systemd[1]: Started Ceph grafana.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:11:12.869 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2024-04-22T15:11:12.869 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2024-04-22T15:11:12.869 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2024-04-22T15:11:12.870 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2024-04-22T15:11:12.870 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2024-04-22T15:11:12.870 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-04-22T15:11:12.870 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2024-04-22T15:11:12.870 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2024-04-22T15:11:12.870 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2024-04-22T15:11:12.870 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2024-04-22T15:11:12.870 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2024-04-22T15:11:12.870 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2024-04-22T15:11:12.870 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2024-04-22T15:11:12.870 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2024-04-22T15:11:12.870 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2024-04-22T15:11:12.870 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2024-04-22T15:11:12.871 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2024-04-22T15:11:12.871 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2024-04-22T15:11:12.871 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2024-04-22T15:11:12.871 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2024-04-22T15:11:12.871 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2024-04-22T15:11:12.871 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2024-04-22T15:11:12.871 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2024-04-22T15:11:12.871 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2024-04-22T15:11:12.871 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2024-04-22T15:11:12.871 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2024-04-22T15:11:12.871 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2024-04-22T15:11:12.871 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2024-04-22T15:11:12.871 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2024-04-22T15:11:12.871 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2024-04-22T15:11:12.872 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2024-04-22T15:11:12.872 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2024-04-22T15:11:12.872 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-04-22T15:11:12.872 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2024-04-22T15:11:12.872 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-04-22T15:11:12.872 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-04-22T15:11:12.872 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2024-04-22T15:11:12.872 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2024-04-22T15:11:12.872 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2024-04-22T15:11:12.872 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2024-04-22T15:11:12.872 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2024-04-22T15:11:12.872 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2024-04-22T15:11:12.872 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2024-04-22T15:11:12.872 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2024-04-22T15:11:12.873 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2024-04-22T15:11:12.873 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2024-04-22T15:11:12.873 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2024-04-22T15:11:12.873 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2024-04-22T15:11:12.873 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2024-04-22T15:11:12.873 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2024-04-22T15:11:12.873 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2024-04-22T15:11:12.873 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2024-04-22T15:11:12.873 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-04-22T15:11:12.873 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2024-04-22T15:11:12.873 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2024-04-22T15:11:12.873 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2024-04-22T15:11:12.873 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2024-04-22T15:11:12.874 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2024-04-22T15:11:12.874 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2024-04-22T15:11:12.874 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2024-04-22T15:11:12.874 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2024-04-22T15:11:12.874 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2024-04-22T15:11:12.874 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2024-04-22T15:11:12.874 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2024-04-22T15:11:12.874 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2024-04-22T15:11:12.874 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2024-04-22T15:11:12.874 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2024-04-22T15:11:12.874 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2024-04-22T15:11:12.874 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2024-04-22T15:11:12.874 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2024-04-22T15:11:12.874 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2024-04-22T15:11:12.875 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2024-04-22T15:11:12.875 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2024-04-22T15:11:12.875 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2024-04-22T15:11:12.875 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2024-04-22T15:11:12.875 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2024-04-22T15:11:12.875 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2024-04-22T15:11:12.875 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2024-04-22T15:11:13.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:13 smithi145 ceph-mon[21899]: osdmap e64: 8 total, 8 up, 8 in 2024-04-22T15:11:13.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:13 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2866091961' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-22T15:11:13.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:13 smithi145 ceph-mon[21899]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-22T15:11:13.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:13 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:13.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:13 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:11:13.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:13 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:13.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:13 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:11:13.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:13 smithi145 ceph-mon[21899]: pgmap v52: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 2.0 KiB/s wr, 3 op/s 2024-04-22T15:11:13.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:13 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:13.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:13 smithi145 ceph-mon[26606]: osdmap e64: 8 total, 8 up, 8 in 2024-04-22T15:11:13.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:13 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2866091961' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-22T15:11:13.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:13 smithi145 ceph-mon[26606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-22T15:11:13.114 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-mon[22747]: osdmap e64: 8 total, 8 up, 8 in 2024-04-22T15:11:13.114 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2866091961' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-22T15:11:13.114 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-mon[22747]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-22T15:11:13.114 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:13.114 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:11:13.114 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:13.114 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:11:13.114 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-mon[22747]: pgmap v52: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 2.0 KiB/s wr, 3 op/s 2024-04-22T15:11:13.114 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:13.115 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2024-04-22T15:11:13.115 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2024-04-22T15:11:13.115 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2024-04-22T15:11:13.115 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2024-04-22T15:11:13.115 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2024-04-22T15:11:13.115 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2024-04-22T15:11:13.116 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2024-04-22T15:11:13.116 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2024-04-22T15:11:13.116 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-04-22T15:11:13.116 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2024-04-22T15:11:13.116 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2024-04-22T15:11:13.116 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2024-04-22T15:11:13.116 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2024-04-22T15:11:13.116 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2024-04-22T15:11:13.116 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-04-22T15:11:13.116 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2024-04-22T15:11:13.116 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2024-04-22T15:11:13.116 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2024-04-22T15:11:13.116 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2024-04-22T15:11:13.116 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-04-22T15:11:13.117 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2024-04-22T15:11:13.117 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2024-04-22T15:11:13.117 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2024-04-22T15:11:13.117 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2024-04-22T15:11:13.117 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2024-04-22T15:11:13.117 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2024-04-22T15:11:13.117 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2024-04-22T15:11:13.117 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2024-04-22T15:11:13.117 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2024-04-22T15:11:13.117 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2024-04-22T15:11:13.117 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2024-04-22T15:11:13.117 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2024-04-22T15:11:13.117 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2024-04-22T15:11:13.118 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2024-04-22T15:11:13.118 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2024-04-22T15:11:13.118 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2024-04-22T15:11:13.118 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2024-04-22T15:11:13.118 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2024-04-22T15:11:13.118 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2024-04-22T15:11:13.118 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2024-04-22T15:11:13.118 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2024-04-22T15:11:13.118 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-04-22T15:11:13.118 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-04-22T15:11:13.118 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-04-22T15:11:13.118 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2024-04-22T15:11:13.119 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-04-22T15:11:13.119 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2024-04-22T15:11:13.119 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2024-04-22T15:11:13.119 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2024-04-22T15:11:13.119 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2024-04-22T15:11:13.119 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2024-04-22T15:11:13.119 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2024-04-22T15:11:13.119 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2024-04-22T15:11:13.119 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2024-04-22T15:11:13.119 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2024-04-22T15:11:13.119 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2024-04-22T15:11:13.119 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2024-04-22T15:11:13.119 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2024-04-22T15:11:13.119 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2024-04-22T15:11:13.120 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2024-04-22T15:11:13.120 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-04-22T15:11:13.120 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2024-04-22T15:11:13.120 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2024-04-22T15:11:13.120 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2024-04-22T15:11:13.120 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2024-04-22T15:11:13.120 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2024-04-22T15:11:13.120 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2024-04-22T15:11:13.120 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2024-04-22T15:11:13.120 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2024-04-22T15:11:13.120 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2024-04-22T15:11:13.121 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2024-04-22T15:11:13.121 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2024-04-22T15:11:13.121 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2024-04-22T15:11:13.121 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2024-04-22T15:11:13.121 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2024-04-22T15:11:13.121 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2024-04-22T15:11:13.121 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2024-04-22T15:11:13.121 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2024-04-22T15:11:13.121 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2024-04-22T15:11:13.121 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2024-04-22T15:11:13.121 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2024-04-22T15:11:13.121 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2024-04-22T15:11:13.121 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2024-04-22T15:11:13.122 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-04-22T15:11:13.122 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-04-22T15:11:13.122 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2024-04-22T15:11:13.122 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-04-22T15:11:13.122 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2024-04-22T15:11:13.122 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2024-04-22T15:11:13.122 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2024-04-22T15:11:13.122 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2024-04-22T15:11:13.122 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2024-04-22T15:11:13.122 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2024-04-22T15:11:13.122 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2024-04-22T15:11:13.122 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2024-04-22T15:11:13.122 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2024-04-22T15:11:13.122 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2024-04-22T15:11:13.123 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2024-04-22T15:11:13.123 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2024-04-22T15:11:13.124 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2024-04-22T15:11:13.124 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-04-22T15:11:13.124 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-04-22T15:11:13.124 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2024-04-22T15:11:13.124 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2024-04-22T15:11:13.124 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2024-04-22T15:11:13.124 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2024-04-22T15:11:13.124 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2024-04-22T15:11:13.124 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2024-04-22T15:11:13.124 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-04-22T15:11:13.124 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2024-04-22T15:11:13.124 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2024-04-22T15:11:13.124 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2024-04-22T15:11:13.125 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2024-04-22T15:11:13.125 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2024-04-22T15:11:13.125 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2024-04-22T15:11:13.125 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2024-04-22T15:11:13.125 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2024-04-22T15:11:13.125 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2024-04-22T15:11:13.125 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2024-04-22T15:11:13.125 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2024-04-22T15:11:13.125 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2024-04-22T15:11:13.125 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2024-04-22T15:11:13.125 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2024-04-22T15:11:13.125 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2024-04-22T15:11:13.125 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2024-04-22T15:11:13.125 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2024-04-22T15:11:13.126 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2024-04-22T15:11:13.126 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2024-04-22T15:11:13.126 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2024-04-22T15:11:13.126 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2024-04-22T15:11:13.126 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2024-04-22T15:11:13.126 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2024-04-22T15:11:13.126 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2024-04-22T15:11:13.126 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2024-04-22T15:11:13.126 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2024-04-22T15:11:13.126 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2024-04-22T15:11:13.126 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2024-04-22T15:11:13.126 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-04-22T15:11:13.126 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2024-04-22T15:11:13.127 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2024-04-22T15:11:13.127 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2024-04-22T15:11:13.127 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2024-04-22T15:11:13.127 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2024-04-22T15:11:13.127 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-04-22T15:11:13.127 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2024-04-22T15:11:13.127 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2024-04-22T15:11:13.127 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2024-04-22T15:11:13.127 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2024-04-22T15:11:13.127 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2024-04-22T15:11:13.127 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2024-04-22T15:11:13.127 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2024-04-22T15:11:13.127 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2024-04-22T15:11:13.127 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2024-04-22T15:11:13.128 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2024-04-22T15:11:13.128 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2024-04-22T15:11:13.128 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2024-04-22T15:11:13.128 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2024-04-22T15:11:13.128 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2024-04-22T15:11:13.128 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2024-04-22T15:11:13.128 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2024-04-22T15:11:13.128 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2024-04-22T15:11:13.128 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2024-04-22T15:11:13.128 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2024-04-22T15:11:13.129 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2024-04-22T15:11:13.129 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2024-04-22T15:11:13.129 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2024-04-22T15:11:13.129 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2024-04-22T15:11:13.129 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2024-04-22T15:11:13.129 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2024-04-22T15:11:13.129 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2024-04-22T15:11:13.129 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2024-04-22T15:11:13.129 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2024-04-22T15:11:13.129 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2024-04-22T15:11:13.130 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2024-04-22T15:11:13.130 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-04-22T15:11:13.130 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-04-22T15:11:13.130 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2024-04-22T15:11:13.130 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2024-04-22T15:11:13.130 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2024-04-22T15:11:13.130 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-04-22T15:11:13.130 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2024-04-22T15:11:13.130 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2024-04-22T15:11:13.130 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-04-22T15:11:13.130 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2024-04-22T15:11:13.130 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2024-04-22T15:11:13.130 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2024-04-22T15:11:13.131 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-04-22T15:11:13.131 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2024-04-22T15:11:13.131 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2024-04-22T15:11:13.131 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2024-04-22T15:11:13.131 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2024-04-22T15:11:13.131 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-04-22T15:11:13.337 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:13 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:13.338 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:13 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:11:13.338 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:13 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:13.338 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:13 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:11:13.338 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:13 smithi145 ceph-mon[26606]: pgmap v52: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 2.0 KiB/s wr, 3 op/s 2024-04-22T15:11:13.338 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:13 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:13.370 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2024-04-22T15:11:13.370 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2024-04-22T15:11:13.370 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2024-04-22T15:11:13.370 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2024-04-22T15:11:13.370 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2024-04-22T15:11:13.370 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-04-22T15:11:13.370 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2024-04-22T15:11:13.370 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2024-04-22T15:11:13.370 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-04-22T15:11:13.371 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2024-04-22T15:11:13.371 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-04-22T15:11:13.371 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-04-22T15:11:13.371 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2024-04-22T15:11:13.371 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2024-04-22T15:11:13.371 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2024-04-22T15:11:13.371 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2024-04-22T15:11:13.371 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2024-04-22T15:11:13.371 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2024-04-22T15:11:13.372 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-04-22T15:11:13.372 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2024-04-22T15:11:13.372 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2024-04-22T15:11:13.372 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2024-04-22T15:11:13.372 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2024-04-22T15:11:13.372 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2024-04-22T15:11:13.372 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2024-04-22T15:11:13.372 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2024-04-22T15:11:13.372 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2024-04-22T15:11:13.373 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2024-04-22T15:11:13.373 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2024-04-22T15:11:13.373 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2024-04-22T15:11:13.373 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2024-04-22T15:11:13.373 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2024-04-22T15:11:13.373 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2024-04-22T15:11:13.373 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2024-04-22T15:11:13.373 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2024-04-22T15:11:13.373 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2024-04-22T15:11:13.373 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2024-04-22T15:11:13.373 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2024-04-22T15:11:13.373 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2024-04-22T15:11:13.373 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2024-04-22T15:11:13.374 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2024-04-22T15:11:13.374 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2024-04-22T15:11:13.374 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2024-04-22T15:11:13.374 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2024-04-22T15:11:13.374 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2024-04-22T15:11:13.374 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2024-04-22T15:11:13.374 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2024-04-22T15:11:13.374 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2024-04-22T15:11:13.374 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2024-04-22T15:11:13.374 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2024-04-22T15:11:13.374 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2024-04-22T15:11:13.374 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2024-04-22T15:11:13.375 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2024-04-22T15:11:13.375 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2024-04-22T15:11:13.375 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2024-04-22T15:11:13.375 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2024-04-22T15:11:13.375 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2024-04-22T15:11:13.375 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2024-04-22T15:11:13.375 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2024-04-22T15:11:13.375 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2024-04-22T15:11:13.375 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2024-04-22T15:11:13.375 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2024-04-22T15:11:13.375 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=573.329711ms 2024-04-22T15:11:13.375 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-04-22T15:11:13.375 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-04-22T15:11:13.375 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-04-22T15:11:13.376 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-04-22T15:11:13.376 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-04-22T15:11:13.376 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-04-22T15:11:13.376 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-04-22T15:11:13.376 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-04-22T15:11:13.376 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-04-22T15:11:13.376 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-04-22T15:11:13.376 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:11:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:11:13+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-04-22T15:11:14.324 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:14 smithi145 ceph-mon[21899]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-04-22T15:11:14.324 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:14 smithi145 ceph-mon[21899]: osdmap e65: 8 total, 8 up, 8 in 2024-04-22T15:11:14.324 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:14 smithi145 ceph-mon[26606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-04-22T15:11:14.324 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:14 smithi145 ceph-mon[26606]: osdmap e65: 8 total, 8 up, 8 in 2024-04-22T15:11:14.426 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:14 smithi172 ceph-mon[22747]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-04-22T15:11:14.426 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:14 smithi172 ceph-mon[22747]: osdmap e65: 8 total, 8 up, 8 in 2024-04-22T15:11:15.204 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:15 smithi172 ceph-mon[22747]: osdmap e66: 8 total, 8 up, 8 in 2024-04-22T15:11:15.204 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:15 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2866091961' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-22T15:11:15.205 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:15 smithi172 ceph-mon[22747]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-22T15:11:15.205 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:15 smithi172 ceph-mon[22747]: pgmap v55: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.7 KiB/s wr, 8 op/s 2024-04-22T15:11:15.205 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:15 smithi172 ceph-mon[22747]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-04-22T15:11:15.205 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:15 smithi172 ceph-mon[22747]: osdmap e67: 8 total, 8 up, 8 in 2024-04-22T15:11:15.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:15 smithi145 ceph-mon[26606]: osdmap e66: 8 total, 8 up, 8 in 2024-04-22T15:11:15.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:15 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2866091961' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-22T15:11:15.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:15 smithi145 ceph-mon[26606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-22T15:11:15.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:15 smithi145 ceph-mon[26606]: pgmap v55: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.7 KiB/s wr, 8 op/s 2024-04-22T15:11:15.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:15 smithi145 ceph-mon[26606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-04-22T15:11:15.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:15 smithi145 ceph-mon[26606]: osdmap e67: 8 total, 8 up, 8 in 2024-04-22T15:11:15.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:15 smithi145 ceph-mon[21899]: osdmap e66: 8 total, 8 up, 8 in 2024-04-22T15:11:15.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:15 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2866091961' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-22T15:11:15.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:15 smithi145 ceph-mon[21899]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-22T15:11:15.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:15 smithi145 ceph-mon[21899]: pgmap v55: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.7 KiB/s wr, 8 op/s 2024-04-22T15:11:15.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:15 smithi145 ceph-mon[21899]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-04-22T15:11:15.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:15 smithi145 ceph-mon[21899]: osdmap e67: 8 total, 8 up, 8 in 2024-04-22T15:11:16.852 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:16 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:16.852 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:16 smithi145 ceph-mon[21899]: osdmap e68: 8 total, 8 up, 8 in 2024-04-22T15:11:16.852 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:16 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2083848055' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-22T15:11:16.853 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:16 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:16.853 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:16 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:16.853 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:16 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:16.853 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:16 smithi145 ceph-mon[26606]: osdmap e68: 8 total, 8 up, 8 in 2024-04-22T15:11:16.853 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:16 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2083848055' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-22T15:11:16.853 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:16 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:16.853 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:16 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:16.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:16 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:16.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:16 smithi172 ceph-mon[22747]: osdmap e68: 8 total, 8 up, 8 in 2024-04-22T15:11:16.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:16 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2083848055' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-22T15:11:16.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:16 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:16.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:16 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:17.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:17 smithi172 ceph-mon[22747]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-22T15:11:17.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:17 smithi172 ceph-mon[22747]: Reconfiguring daemon alertmanager.a on smithi145 2024-04-22T15:11:17.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:17 smithi172 ceph-mon[22747]: pgmap v58: 129 pgs: 64 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-04-22T15:11:17.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:17 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2083848055' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-04-22T15:11:17.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:17 smithi172 ceph-mon[22747]: osdmap e69: 8 total, 8 up, 8 in 2024-04-22T15:11:17.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:17 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2083848055' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-22T15:11:17.975 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:17 smithi145 ceph-mon[21899]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-22T15:11:17.975 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:17 smithi145 ceph-mon[21899]: Reconfiguring daemon alertmanager.a on smithi145 2024-04-22T15:11:17.975 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:17 smithi145 ceph-mon[21899]: pgmap v58: 129 pgs: 64 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-04-22T15:11:17.975 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:17 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2083848055' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-04-22T15:11:17.975 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:17 smithi145 ceph-mon[21899]: osdmap e69: 8 total, 8 up, 8 in 2024-04-22T15:11:17.975 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:17 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2083848055' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-22T15:11:17.975 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:17 smithi145 ceph-mon[26606]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-22T15:11:17.975 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:17 smithi145 ceph-mon[26606]: Reconfiguring daemon alertmanager.a on smithi145 2024-04-22T15:11:17.975 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:17 smithi145 ceph-mon[26606]: pgmap v58: 129 pgs: 64 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-04-22T15:11:17.975 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:17 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2083848055' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-04-22T15:11:17.976 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:17 smithi145 ceph-mon[26606]: osdmap e69: 8 total, 8 up, 8 in 2024-04-22T15:11:17.976 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:17 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2083848055' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-22T15:11:18.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:11:17 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:11:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:11:19.146 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:11:19.147 INFO:teuthology.orchestra.run.smithi145.stdout: "id": "ef7205f6-b0d5-4b0d-8441-5820290a20e9", 2024-04-22T15:11:19.147 INFO:teuthology.orchestra.run.smithi145.stdout: "epoch": 1, 2024-04-22T15:11:19.147 INFO:teuthology.orchestra.run.smithi145.stdout: "predecessor_uuid": "9716de0c-e911-4ff5-b420-06054021de9e", 2024-04-22T15:11:19.147 INFO:teuthology.orchestra.run.smithi145.stdout: "sync_status": [], 2024-04-22T15:11:19.147 INFO:teuthology.orchestra.run.smithi145.stdout: "period_map": { 2024-04-22T15:11:19.147 INFO:teuthology.orchestra.run.smithi145.stdout: "id": "ef7205f6-b0d5-4b0d-8441-5820290a20e9", 2024-04-22T15:11:19.147 INFO:teuthology.orchestra.run.smithi145.stdout: "zonegroups": [ 2024-04-22T15:11:19.147 INFO:teuthology.orchestra.run.smithi145.stdout: { 2024-04-22T15:11:19.147 INFO:teuthology.orchestra.run.smithi145.stdout: "id": "f46bedf2-002a-4b17-8d46-37ed97c44cee", 2024-04-22T15:11:19.147 INFO:teuthology.orchestra.run.smithi145.stdout: "name": "default", 2024-04-22T15:11:19.147 INFO:teuthology.orchestra.run.smithi145.stdout: "api_name": "default", 2024-04-22T15:11:19.147 INFO:teuthology.orchestra.run.smithi145.stdout: "is_master": "true", 2024-04-22T15:11:19.147 INFO:teuthology.orchestra.run.smithi145.stdout: "endpoints": [], 2024-04-22T15:11:19.147 INFO:teuthology.orchestra.run.smithi145.stdout: "hostnames": [], 2024-04-22T15:11:19.148 INFO:teuthology.orchestra.run.smithi145.stdout: "hostnames_s3website": [], 2024-04-22T15:11:19.148 INFO:teuthology.orchestra.run.smithi145.stdout: "master_zone": "f1401a66-b907-4f13-b8a7-1a3c48a44285", 2024-04-22T15:11:19.148 INFO:teuthology.orchestra.run.smithi145.stdout: "zones": [ 2024-04-22T15:11:19.148 INFO:teuthology.orchestra.run.smithi145.stdout: { 2024-04-22T15:11:19.148 INFO:teuthology.orchestra.run.smithi145.stdout: "id": "f1401a66-b907-4f13-b8a7-1a3c48a44285", 2024-04-22T15:11:19.148 INFO:teuthology.orchestra.run.smithi145.stdout: "name": "z", 2024-04-22T15:11:19.148 INFO:teuthology.orchestra.run.smithi145.stdout: "endpoints": [], 2024-04-22T15:11:19.148 INFO:teuthology.orchestra.run.smithi145.stdout: "log_meta": "false", 2024-04-22T15:11:19.148 INFO:teuthology.orchestra.run.smithi145.stdout: "log_data": "false", 2024-04-22T15:11:19.148 INFO:teuthology.orchestra.run.smithi145.stdout: "bucket_index_max_shards": 11, 2024-04-22T15:11:19.148 INFO:teuthology.orchestra.run.smithi145.stdout: "read_only": "false", 2024-04-22T15:11:19.148 INFO:teuthology.orchestra.run.smithi145.stdout: "tier_type": "", 2024-04-22T15:11:19.148 INFO:teuthology.orchestra.run.smithi145.stdout: "sync_from_all": "true", 2024-04-22T15:11:19.148 INFO:teuthology.orchestra.run.smithi145.stdout: "sync_from": [], 2024-04-22T15:11:19.148 INFO:teuthology.orchestra.run.smithi145.stdout: "redirect_zone": "" 2024-04-22T15:11:19.148 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:11:19.148 INFO:teuthology.orchestra.run.smithi145.stdout: ], 2024-04-22T15:11:19.149 INFO:teuthology.orchestra.run.smithi145.stdout: "placement_targets": [ 2024-04-22T15:11:19.149 INFO:teuthology.orchestra.run.smithi145.stdout: { 2024-04-22T15:11:19.149 INFO:teuthology.orchestra.run.smithi145.stdout: "name": "default-placement", 2024-04-22T15:11:19.149 INFO:teuthology.orchestra.run.smithi145.stdout: "tags": [], 2024-04-22T15:11:19.149 INFO:teuthology.orchestra.run.smithi145.stdout: "storage_classes": [ 2024-04-22T15:11:19.149 INFO:teuthology.orchestra.run.smithi145.stdout: "STANDARD" 2024-04-22T15:11:19.149 INFO:teuthology.orchestra.run.smithi145.stdout: ] 2024-04-22T15:11:19.149 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:11:19.149 INFO:teuthology.orchestra.run.smithi145.stdout: ], 2024-04-22T15:11:19.149 INFO:teuthology.orchestra.run.smithi145.stdout: "default_placement": "default-placement", 2024-04-22T15:11:19.149 INFO:teuthology.orchestra.run.smithi145.stdout: "realm_id": "e694c198-925d-4328-a804-67b7d03db749", 2024-04-22T15:11:19.149 INFO:teuthology.orchestra.run.smithi145.stdout: "sync_policy": { 2024-04-22T15:11:19.149 INFO:teuthology.orchestra.run.smithi145.stdout: "groups": [] 2024-04-22T15:11:19.149 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:11:19.149 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:11:19.149 INFO:teuthology.orchestra.run.smithi145.stdout: ], 2024-04-22T15:11:19.150 INFO:teuthology.orchestra.run.smithi145.stdout: "short_zone_ids": [ 2024-04-22T15:11:19.150 INFO:teuthology.orchestra.run.smithi145.stdout: { 2024-04-22T15:11:19.150 INFO:teuthology.orchestra.run.smithi145.stdout: "key": "f1401a66-b907-4f13-b8a7-1a3c48a44285", 2024-04-22T15:11:19.150 INFO:teuthology.orchestra.run.smithi145.stdout: "val": 1184713553 2024-04-22T15:11:19.150 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:11:19.150 INFO:teuthology.orchestra.run.smithi145.stdout: ] 2024-04-22T15:11:19.150 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:11:19.150 INFO:teuthology.orchestra.run.smithi145.stdout: "master_zonegroup": "f46bedf2-002a-4b17-8d46-37ed97c44cee", 2024-04-22T15:11:19.150 INFO:teuthology.orchestra.run.smithi145.stdout: "master_zone": "f1401a66-b907-4f13-b8a7-1a3c48a44285", 2024-04-22T15:11:19.150 INFO:teuthology.orchestra.run.smithi145.stdout: "period_config": { 2024-04-22T15:11:19.150 INFO:teuthology.orchestra.run.smithi145.stdout: "bucket_quota": { 2024-04-22T15:11:19.150 INFO:teuthology.orchestra.run.smithi145.stdout: "enabled": false, 2024-04-22T15:11:19.150 INFO:teuthology.orchestra.run.smithi145.stdout: "check_on_raw": false, 2024-04-22T15:11:19.150 INFO:teuthology.orchestra.run.smithi145.stdout: "max_size": -1, 2024-04-22T15:11:19.150 INFO:teuthology.orchestra.run.smithi145.stdout: "max_size_kb": 0, 2024-04-22T15:11:19.151 INFO:teuthology.orchestra.run.smithi145.stdout: "max_objects": -1 2024-04-22T15:11:19.151 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:11:19.151 INFO:teuthology.orchestra.run.smithi145.stdout: "user_quota": { 2024-04-22T15:11:19.151 INFO:teuthology.orchestra.run.smithi145.stdout: "enabled": false, 2024-04-22T15:11:19.151 INFO:teuthology.orchestra.run.smithi145.stdout: "check_on_raw": false, 2024-04-22T15:11:19.151 INFO:teuthology.orchestra.run.smithi145.stdout: "max_size": -1, 2024-04-22T15:11:19.151 INFO:teuthology.orchestra.run.smithi145.stdout: "max_size_kb": 0, 2024-04-22T15:11:19.151 INFO:teuthology.orchestra.run.smithi145.stdout: "max_objects": -1 2024-04-22T15:11:19.151 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:11:19.151 INFO:teuthology.orchestra.run.smithi145.stdout: "user_ratelimit": { 2024-04-22T15:11:19.151 INFO:teuthology.orchestra.run.smithi145.stdout: "max_read_ops": 0, 2024-04-22T15:11:19.151 INFO:teuthology.orchestra.run.smithi145.stdout: "max_write_ops": 0, 2024-04-22T15:11:19.151 INFO:teuthology.orchestra.run.smithi145.stdout: "max_read_bytes": 0, 2024-04-22T15:11:19.151 INFO:teuthology.orchestra.run.smithi145.stdout: "max_write_bytes": 0, 2024-04-22T15:11:19.151 INFO:teuthology.orchestra.run.smithi145.stdout: "enabled": false 2024-04-22T15:11:19.152 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:11:19.152 INFO:teuthology.orchestra.run.smithi145.stdout: "bucket_ratelimit": { 2024-04-22T15:11:19.152 INFO:teuthology.orchestra.run.smithi145.stdout: "max_read_ops": 0, 2024-04-22T15:11:19.152 INFO:teuthology.orchestra.run.smithi145.stdout: "max_write_ops": 0, 2024-04-22T15:11:19.152 INFO:teuthology.orchestra.run.smithi145.stdout: "max_read_bytes": 0, 2024-04-22T15:11:19.152 INFO:teuthology.orchestra.run.smithi145.stdout: "max_write_bytes": 0, 2024-04-22T15:11:19.152 INFO:teuthology.orchestra.run.smithi145.stdout: "enabled": false 2024-04-22T15:11:19.152 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:11:19.152 INFO:teuthology.orchestra.run.smithi145.stdout: "anonymous_ratelimit": { 2024-04-22T15:11:19.152 INFO:teuthology.orchestra.run.smithi145.stdout: "max_read_ops": 0, 2024-04-22T15:11:19.152 INFO:teuthology.orchestra.run.smithi145.stdout: "max_write_ops": 0, 2024-04-22T15:11:19.152 INFO:teuthology.orchestra.run.smithi145.stdout: "max_read_bytes": 0, 2024-04-22T15:11:19.152 INFO:teuthology.orchestra.run.smithi145.stdout: "max_write_bytes": 0, 2024-04-22T15:11:19.152 INFO:teuthology.orchestra.run.smithi145.stdout: "enabled": false 2024-04-22T15:11:19.152 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:11:19.152 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:11:19.152 INFO:teuthology.orchestra.run.smithi145.stdout: "realm_id": "e694c198-925d-4328-a804-67b7d03db749", 2024-04-22T15:11:19.153 INFO:teuthology.orchestra.run.smithi145.stdout: "realm_name": "r", 2024-04-22T15:11:19.153 INFO:teuthology.orchestra.run.smithi145.stdout: "realm_epoch": 2 2024-04-22T15:11:19.153 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:11:19.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:19 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2083848055' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-04-22T15:11:19.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:19 smithi145 ceph-mon[21899]: osdmap e70: 8 total, 8 up, 8 in 2024-04-22T15:11:19.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:19 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2083848055' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-22T15:11:19.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:19 smithi145 ceph-mon[21899]: pgmap v61: 129 pgs: 64 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:19.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:19 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2083848055' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-04-22T15:11:19.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:19 smithi145 ceph-mon[26606]: osdmap e70: 8 total, 8 up, 8 in 2024-04-22T15:11:19.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:19 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2083848055' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-22T15:11:19.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:19 smithi145 ceph-mon[26606]: pgmap v61: 129 pgs: 64 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:19.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:19 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2083848055' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-04-22T15:11:19.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:19 smithi172 ceph-mon[22747]: osdmap e70: 8 total, 8 up, 8 in 2024-04-22T15:11:19.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:19 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2083848055' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-22T15:11:19.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:19 smithi172 ceph-mon[22747]: pgmap v61: 129 pgs: 64 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:19.900 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000' 2024-04-22T15:11:20.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:20 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2083848055' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-04-22T15:11:20.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:20 smithi145 ceph-mon[26606]: osdmap e71: 8 total, 8 up, 8 in 2024-04-22T15:11:20.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:20 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2083848055' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-04-22T15:11:20.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:20 smithi145 ceph-mon[21899]: osdmap e71: 8 total, 8 up, 8 in 2024-04-22T15:11:20.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:20 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2083848055' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-04-22T15:11:20.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:20 smithi172 ceph-mon[22747]: osdmap e71: 8 total, 8 up, 8 in 2024-04-22T15:11:20.743 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:11:20 smithi145 systemd[1]: Stopping Ceph alertmanager.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:11:20.743 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:11:20 smithi145 bash[46492]: Error: no container with name or ID "ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager.a" found: no such container 2024-04-22T15:11:21.072 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:11:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[44942]: level=info ts=2024-04-22T15:11:20.952Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-04-22T15:11:21.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:21 smithi145 ceph-mon[21899]: pgmap v63: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 208 B/s rd, 417 B/s wr, 1 op/s 2024-04-22T15:11:21.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:21 smithi145 ceph-mon[26606]: pgmap v63: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 208 B/s rd, 417 B/s wr, 1 op/s 2024-04-22T15:11:21.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:21 smithi172 ceph-mon[22747]: pgmap v63: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 208 B/s rd, 417 B/s wr, 1 op/s 2024-04-22T15:11:21.732 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:11:21 smithi145 podman[46517]: 2024-04-22 15:11:21.403026049 +0000 UTC m=+0.757470258 container died 61c101b04b81af627f21b73bbd83549be8027b44257db7909141c4105de5836b (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T15:11:21.733 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:11:21 smithi145 podman[46517]: 2024-04-22 15:11:21.73021255 +0000 UTC m=+1.084656765 container cleanup 61c101b04b81af627f21b73bbd83549be8027b44257db7909141c4105de5836b (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T15:11:21.992 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:11:21 smithi145 bash[46517]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a 2024-04-22T15:11:21.992 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:11:21 smithi145 bash[46561]: Error: no container with name or ID "ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager.a" found: no such container 2024-04-22T15:11:21.992 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:11:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:11:21] "GET /metrics HTTP/1.1" 200 194131 "" "Prometheus/2.33.4" 2024-04-22T15:11:22.245 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:11:21 smithi145 podman[46536]: 2024-04-22 15:11:21.98955847 +0000 UTC m=+1.015749346 container remove 61c101b04b81af627f21b73bbd83549be8027b44257db7909141c4105de5836b (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T15:11:22.245 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:11:22 smithi145 podman[46536]: 2024-04-22 15:11:22.073700799 +0000 UTC m=+1.099891683 volume remove e3f30f112e296b24915b9316c00697391643fdde220f7775c054becbc0af3b71 2024-04-22T15:11:22.245 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:11:22 smithi145 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-22T15:11:22.245 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:11:22 smithi145 systemd[1]: Stopped Ceph alertmanager.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:11:22.245 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:11:22 smithi145 systemd[1]: Starting Ceph alertmanager.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:11:23.326 INFO:teuthology.orchestra.run.smithi145.stdout:Scheduled rgw.foo update... 2024-04-22T15:11:23.560 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:11:23 smithi145 podman[46656]: 2024-04-22 15:11:23.123493618 +0000 UTC m=+0.099175981 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-04-22T15:11:23.561 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:11:23 smithi145 podman[46656]: 2024-04-22 15:11:23.240661773 +0000 UTC m=+0.216344111 volume create 2a003ea9672060f2d287c43633e2d33b4877f2624de207e76dfabf83f212d7f2 2024-04-22T15:11:23.561 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:11:23 smithi145 podman[46656]: 2024-04-22 15:11:23.339259459 +0000 UTC m=+0.314941795 container create 9cbf618f121210121a6236180d6f9acf72362c6c023ade92234d7495e5d5fb91 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T15:11:24.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:24 smithi145 ceph-mon[21899]: pgmap v64: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-04-22T15:11:24.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:24 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:24.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:24 smithi145 ceph-mon[26606]: pgmap v64: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-04-22T15:11:24.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:24 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:24.373 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:11:23 smithi145 podman[46656]: 2024-04-22 15:11:23.976416964 +0000 UTC m=+0.952099299 container init 9cbf618f121210121a6236180d6f9acf72362c6c023ade92234d7495e5d5fb91 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T15:11:24.374 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:11:23 smithi145 podman[46656]: 2024-04-22 15:11:23.980023901 +0000 UTC m=+0.955706237 container start 9cbf618f121210121a6236180d6f9acf72362c6c023ade92234d7495e5d5fb91 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T15:11:24.374 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:11:23 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=info ts=2024-04-22T15:11:23.991Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-04-22T15:11:24.374 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:11:23 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=info ts=2024-04-22T15:11:23.991Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-04-22T15:11:24.374 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:11:23 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=info ts=2024-04-22T15:11:23.993Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.145 port=9094 2024-04-22T15:11:24.374 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:11:23 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=info ts=2024-04-22T15:11:23.994Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-22T15:11:24.374 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:11:24 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=info ts=2024-04-22T15:11:24.019Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-22T15:11:24.374 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:11:24 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=info ts=2024-04-22T15:11:24.019Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-22T15:11:24.374 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:11:24 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=info ts=2024-04-22T15:11:24.022Z caller=main.go:518 msg=Listening address=:9093 2024-04-22T15:11:24.374 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:11:24 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=info ts=2024-04-22T15:11:24.022Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-04-22T15:11:24.374 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:11:24 smithi145 bash[46656]: 9cbf618f121210121a6236180d6f9acf72362c6c023ade92234d7495e5d5fb91 2024-04-22T15:11:24.374 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:11:24 smithi145 systemd[1]: Started Ceph alertmanager.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:11:24.453 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:24 smithi172 ceph-mon[22747]: pgmap v64: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-04-22T15:11:24.453 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:24 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:24.455 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph osd pool create foo' 2024-04-22T15:11:25.032 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:25 smithi172 ceph-mon[22747]: from='client.14688 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:25.032 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:25 smithi172 ceph-mon[22747]: Saving service rgw.foo spec with placement count:2 2024-04-22T15:11:25.032 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:25 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:25.307 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:25 smithi145 ceph-mon[26606]: from='client.14688 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:25.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:25 smithi145 ceph-mon[26606]: Saving service rgw.foo spec with placement count:2 2024-04-22T15:11:25.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:25 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:25.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:25 smithi145 ceph-mon[21899]: from='client.14688 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:25.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:25 smithi145 ceph-mon[21899]: Saving service rgw.foo spec with placement count:2 2024-04-22T15:11:25.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:25 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:26.093 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:26 smithi172 ceph-mon[22747]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T15:11:26.093 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:26 smithi172 ceph-mon[22747]: Reconfiguring daemon prometheus.a on smithi172 2024-04-22T15:11:26.093 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:26 smithi172 ceph-mon[22747]: pgmap v65: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.1 KiB/s wr, 3 op/s 2024-04-22T15:11:26.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:26 smithi145 ceph-mon[21899]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T15:11:26.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:26 smithi145 ceph-mon[21899]: Reconfiguring daemon prometheus.a on smithi172 2024-04-22T15:11:26.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:26 smithi145 ceph-mon[21899]: pgmap v65: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.1 KiB/s wr, 3 op/s 2024-04-22T15:11:26.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:26 smithi145 ceph-mon[26606]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T15:11:26.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:26 smithi145 ceph-mon[26606]: Reconfiguring daemon prometheus.a on smithi172 2024-04-22T15:11:26.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:26 smithi145 ceph-mon[26606]: pgmap v65: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.1 KiB/s wr, 3 op/s 2024-04-22T15:11:26.374 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:11:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=info ts=2024-04-22T15:11:25.995Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000479236s 2024-04-22T15:11:27.051 INFO:teuthology.orchestra.run.smithi145.stderr:pool 'foo' created 2024-04-22T15:11:27.211 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:27 smithi172 systemd[1]: Stopping Ceph prometheus.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:11:27.211 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:27 smithi172 bash[39550]: Error: no container with name or ID "ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-22T15:11:27.211 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:27 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/4050432403' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-22T15:11:27.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:27 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/4050432403' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-22T15:11:27.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:27 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/4050432403' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-22T15:11:27.468 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:27 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[37887]: ts=2024-04-22T15:11:27.208Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-22T15:11:27.468 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:27 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[37887]: ts=2024-04-22T15:11:27.208Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-22T15:11:27.468 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:27 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[37887]: ts=2024-04-22T15:11:27.208Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-22T15:11:27.468 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:27 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[37887]: ts=2024-04-22T15:11:27.208Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-22T15:11:27.468 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:27 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[37887]: ts=2024-04-22T15:11:27.208Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-22T15:11:27.468 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:27 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[37887]: ts=2024-04-22T15:11:27.208Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-22T15:11:27.468 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:27 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[37887]: ts=2024-04-22T15:11:27.208Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-22T15:11:27.468 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:27 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[37887]: ts=2024-04-22T15:11:27.208Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-22T15:11:27.469 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:27 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[37887]: ts=2024-04-22T15:11:27.209Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-22T15:11:27.469 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:27 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[37887]: ts=2024-04-22T15:11:27.209Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-22T15:11:27.469 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:27 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[37887]: ts=2024-04-22T15:11:27.209Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-22T15:11:27.469 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:27 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[37887]: ts=2024-04-22T15:11:27.209Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-22T15:11:27.469 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:27 smithi172 podman[39563]: 2024-04-22 15:11:27.218758986 +0000 UTC m=+0.134750639 container died 6e86e365411f2b6c450ef7669e85c3fa26262d5408830e7e99d76d2d4237bde2 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:11:27.754 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'rbd pool init foo' 2024-04-22T15:11:27.828 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:27 smithi172 podman[39563]: 2024-04-22 15:11:27.591533408 +0000 UTC m=+0.507525066 container cleanup 6e86e365411f2b6c450ef7669e85c3fa26262d5408830e7e99d76d2d4237bde2 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:11:27.828 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:27 smithi172 bash[39563]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a 2024-04-22T15:11:27.828 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:27 smithi172 bash[39590]: Error: no container with name or ID "ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-22T15:11:28.109 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:28 smithi172 ceph-mon[22747]: pgmap v66: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 920 B/s rd, 1.0 KiB/s wr, 2 op/s 2024-04-22T15:11:28.109 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:28 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/4050432403' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-04-22T15:11:28.109 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:28 smithi172 ceph-mon[22747]: osdmap e72: 8 total, 8 up, 8 in 2024-04-22T15:11:28.109 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:27 smithi172 podman[39578]: 2024-04-22 15:11:27.825994604 +0000 UTC m=+0.606684602 container remove 6e86e365411f2b6c450ef7669e85c3fa26262d5408830e7e99d76d2d4237bde2 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:11:28.109 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:27 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-22T15:11:28.109 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:27 smithi172 systemd[1]: Stopped Ceph prometheus.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:11:28.109 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:27 smithi172 systemd[1]: Starting Ceph prometheus.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:11:28.109 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:28 smithi172 podman[39658]: 2024-04-22 15:11:27.967321733 +0000 UTC m=+0.018551765 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-22T15:11:28.109 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:28 smithi172 podman[39658]: 2024-04-22 15:11:28.106491642 +0000 UTC m=+0.157721663 container create 9121b6b1d1de107a43e0fdb307f670fb4116add9e9e4fd48bb09e8c4a4f93e8f (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:11:28.326 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:28 smithi145 ceph-mon[21899]: pgmap v66: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 920 B/s rd, 1.0 KiB/s wr, 2 op/s 2024-04-22T15:11:28.326 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:28 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/4050432403' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-04-22T15:11:28.326 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:28 smithi145 ceph-mon[21899]: osdmap e72: 8 total, 8 up, 8 in 2024-04-22T15:11:28.326 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:28 smithi145 ceph-mon[26606]: pgmap v66: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 920 B/s rd, 1.0 KiB/s wr, 2 op/s 2024-04-22T15:11:28.327 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:28 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/4050432403' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-04-22T15:11:28.327 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:28 smithi145 ceph-mon[26606]: osdmap e72: 8 total, 8 up, 8 in 2024-04-22T15:11:28.407 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:28 smithi172 podman[39658]: 2024-04-22 15:11:28.321000701 +0000 UTC m=+0.372230726 container init 9121b6b1d1de107a43e0fdb307f670fb4116add9e9e4fd48bb09e8c4a4f93e8f (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:11:28.407 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:28 smithi172 podman[39658]: 2024-04-22 15:11:28.324249048 +0000 UTC m=+0.375479075 container start 9121b6b1d1de107a43e0fdb307f670fb4116add9e9e4fd48bb09e8c4a4f93e8f (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:11:28.407 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:28 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:11:28.347Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-22T15:11:28.407 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:28 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:11:28.347Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-22T15:11:28.407 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:28 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:11:28.347Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-22T15:11:28.407 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:28 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:11:28.347Z caller=main.go:518 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi172 (none))" 2024-04-22T15:11:28.407 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:28 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:11:28.347Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-22T15:11:28.407 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:28 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:11:28.347Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-22T15:11:28.407 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:28 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:11:28.349Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-22T15:11:28.408 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:28 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:11:28.349Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-22T15:11:28.408 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:28 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:11:28.350Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-22T15:11:28.408 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:28 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:11:28.351Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-22T15:11:28.408 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:28 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:11:28.351Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.434µs 2024-04-22T15:11:28.408 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:28 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:11:28.351Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-22T15:11:28.408 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:28 smithi172 bash[39658]: 9121b6b1d1de107a43e0fdb307f670fb4116add9e9e4fd48bb09e8c4a4f93e8f 2024-04-22T15:11:28.718 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:28 smithi172 systemd[1]: Started Ceph prometheus.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:11:28.718 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:28 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:11:28.564Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-04-22T15:11:28.718 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:28 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:11:28.565Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-04-22T15:11:28.718 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:28 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:11:28.565Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=23.53µs wal_replay_duration=213.235554ms total_replay_duration=213.271151ms 2024-04-22T15:11:28.718 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:28 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:11:28.566Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-22T15:11:28.718 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:28 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:11:28.566Z caller=main.go:947 level=info msg="TSDB started" 2024-04-22T15:11:28.718 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:28 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:11:28.566Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-22T15:11:28.718 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:28 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:11:28.582Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.385331ms db_storage=989ns remote_storage=1.448µs web_handler=311ns query_engine=728ns scrape=200.383µs scrape_sd=26.811µs notify=24.945µs notify_sd=9.622µs rules=14.88184ms 2024-04-22T15:11:28.718 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:11:28 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:11:28.582Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-22T15:11:29.319 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:29 smithi172 ceph-mon[22747]: osdmap e73: 8 total, 8 up, 8 in 2024-04-22T15:11:29.319 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:29 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:29.319 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:29 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:29.319 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:29 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T15:11:29.319 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:29 smithi172 ceph-mon[22747]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T15:11:29.319 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:29 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.145:9093"}]: dispatch 2024-04-22T15:11:29.319 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:29 smithi172 ceph-mon[22747]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.145:9093"}]: dispatch 2024-04-22T15:11:29.320 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:29 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:29.320 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:29 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:11:29.320 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:29 smithi172 ceph-mon[22747]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:11:29.320 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:29 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.172:9095"}]: dispatch 2024-04-22T15:11:29.320 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:29 smithi172 ceph-mon[22747]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.172:9095"}]: dispatch 2024-04-22T15:11:29.320 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:29 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:29.320 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:29 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T15:11:29.320 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:29 smithi172 ceph-mon[22747]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T15:11:29.320 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:29 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.172:3000"}]: dispatch 2024-04-22T15:11:29.320 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:29 smithi172 ceph-mon[22747]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.172:3000"}]: dispatch 2024-04-22T15:11:29.320 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:29 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:29.320 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:29 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:11:29.320 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:29 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:29.320 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:29 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:11:29.321 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:29 smithi172 ceph-mon[22747]: pgmap v69: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 895 B/s wr, 2 op/s 2024-04-22T15:11:29.368 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[21899]: osdmap e73: 8 total, 8 up, 8 in 2024-04-22T15:11:29.368 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:29.368 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:29.368 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T15:11:29.368 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[21899]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T15:11:29.368 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.145:9093"}]: dispatch 2024-04-22T15:11:29.368 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[21899]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.145:9093"}]: dispatch 2024-04-22T15:11:29.368 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:29.368 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:11:29.368 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[21899]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:11:29.368 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.172:9095"}]: dispatch 2024-04-22T15:11:29.368 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[21899]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.172:9095"}]: dispatch 2024-04-22T15:11:29.368 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:29.369 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T15:11:29.369 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[21899]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T15:11:29.369 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.172:3000"}]: dispatch 2024-04-22T15:11:29.369 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[21899]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.172:3000"}]: dispatch 2024-04-22T15:11:29.369 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:29.369 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:11:29.369 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:29.369 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:11:29.369 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[21899]: pgmap v69: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 895 B/s wr, 2 op/s 2024-04-22T15:11:29.369 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[26606]: osdmap e73: 8 total, 8 up, 8 in 2024-04-22T15:11:29.369 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:29.369 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:29.369 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T15:11:29.370 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[26606]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T15:11:29.370 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.145:9093"}]: dispatch 2024-04-22T15:11:29.370 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[26606]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.145:9093"}]: dispatch 2024-04-22T15:11:29.370 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:29.370 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:11:29.370 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[26606]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:11:29.370 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.172:9095"}]: dispatch 2024-04-22T15:11:29.370 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[26606]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.172:9095"}]: dispatch 2024-04-22T15:11:29.370 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:29.370 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T15:11:29.370 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[26606]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T15:11:29.370 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.172:3000"}]: dispatch 2024-04-22T15:11:29.370 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[26606]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.172:3000"}]: dispatch 2024-04-22T15:11:29.371 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:29.371 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:11:29.371 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:29.371 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:11:29.371 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:29 smithi145 ceph-mon[26606]: pgmap v69: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 895 B/s wr, 2 op/s 2024-04-22T15:11:30.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:30 smithi145 ceph-mon[26606]: osdmap e74: 8 total, 8 up, 8 in 2024-04-22T15:11:30.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:30 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3589408033' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-22T15:11:30.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:30 smithi145 ceph-mon[21899]: osdmap e74: 8 total, 8 up, 8 in 2024-04-22T15:11:30.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:30 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3589408033' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-22T15:11:30.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:30 smithi172 ceph-mon[22747]: osdmap e74: 8 total, 8 up, 8 in 2024-04-22T15:11:30.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:30 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/3589408033' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-22T15:11:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:31 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/3589408033' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-04-22T15:11:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:31 smithi172 ceph-mon[22747]: osdmap e75: 8 total, 8 up, 8 in 2024-04-22T15:11:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:31 smithi172 ceph-mon[22747]: pgmap v72: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:31.772 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:31 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3589408033' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-04-22T15:11:31.772 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:31 smithi145 ceph-mon[26606]: osdmap e75: 8 total, 8 up, 8 in 2024-04-22T15:11:31.772 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:31 smithi145 ceph-mon[26606]: pgmap v72: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:31.772 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:31 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3589408033' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-04-22T15:11:31.772 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:31 smithi145 ceph-mon[21899]: osdmap e75: 8 total, 8 up, 8 in 2024-04-22T15:11:31.772 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:31 smithi145 ceph-mon[21899]: pgmap v72: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:32.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:32 smithi145 ceph-mon[21899]: osdmap e76: 8 total, 8 up, 8 in 2024-04-22T15:11:32.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:32 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:32.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:32 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:32.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:32 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:32.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:32 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:32.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:32 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:32.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:32 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:32.875 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:32 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi145.ygrqmi", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T15:11:32.875 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:32 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi145.ygrqmi", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-22T15:11:32.875 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:32 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:32.875 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:32 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:32.875 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:32 smithi145 ceph-mon[26606]: osdmap e76: 8 total, 8 up, 8 in 2024-04-22T15:11:32.875 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:32 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:32.875 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:32 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:32.875 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:32 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:32.875 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:32 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:32.875 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:32 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:32.875 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:32 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:32.875 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:32 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi145.ygrqmi", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T15:11:32.876 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:32 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi145.ygrqmi", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-22T15:11:32.876 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:32 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:32.876 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:32 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:32.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:32 smithi172 ceph-mon[22747]: osdmap e76: 8 total, 8 up, 8 in 2024-04-22T15:11:32.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:32 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:32.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:32 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:32.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:32 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:32.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:32 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:32.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:32 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:32.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:32 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:32.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:32 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi145.ygrqmi", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T15:11:32.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:32 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi145.ygrqmi", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-22T15:11:32.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:32 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:32.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:32 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:33.438 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch apply iscsi foo u p' 2024-04-22T15:11:33.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:33 smithi145 ceph-mon[21899]: Saving service rgw.foo spec with placement count:2 2024-04-22T15:11:33.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:33 smithi145 ceph-mon[21899]: Deploying daemon rgw.foo.smithi145.ygrqmi on smithi145 2024-04-22T15:11:33.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:33 smithi145 ceph-mon[21899]: osdmap e77: 8 total, 8 up, 8 in 2024-04-22T15:11:33.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:33 smithi145 ceph-mon[21899]: pgmap v75: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:33.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:33 smithi145 ceph-mon[26606]: Saving service rgw.foo spec with placement count:2 2024-04-22T15:11:33.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:33 smithi145 ceph-mon[26606]: Deploying daemon rgw.foo.smithi145.ygrqmi on smithi145 2024-04-22T15:11:33.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:33 smithi145 ceph-mon[26606]: osdmap e77: 8 total, 8 up, 8 in 2024-04-22T15:11:33.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:33 smithi145 ceph-mon[26606]: pgmap v75: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:33.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:33 smithi172 ceph-mon[22747]: Saving service rgw.foo spec with placement count:2 2024-04-22T15:11:33.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:33 smithi172 ceph-mon[22747]: Deploying daemon rgw.foo.smithi145.ygrqmi on smithi145 2024-04-22T15:11:33.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:33 smithi172 ceph-mon[22747]: osdmap e77: 8 total, 8 up, 8 in 2024-04-22T15:11:33.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:33 smithi172 ceph-mon[22747]: pgmap v75: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:34.224 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:11:33 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=info ts=2024-04-22T15:11:33.996Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002200888s 2024-04-22T15:11:36.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:36 smithi145 ceph-mon[21899]: pgmap v76: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 345 B/s wr, 0 op/s 2024-04-22T15:11:36.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:36 smithi145 ceph-mon[26606]: pgmap v76: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 345 B/s wr, 0 op/s 2024-04-22T15:11:36.267 INFO:teuthology.orchestra.run.smithi145.stdout:Scheduled iscsi.foo update... 2024-04-22T15:11:36.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:36 smithi172 ceph-mon[22747]: pgmap v76: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 345 B/s wr, 0 op/s 2024-04-22T15:11:37.016 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'sleep 180' 2024-04-22T15:11:37.528 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:37 smithi145 ceph-mon[26606]: from='client.14706 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:37.528 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:37 smithi145 ceph-mon[26606]: Saving service iscsi.foo spec with placement count:1 2024-04-22T15:11:37.528 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:37 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:37.528 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:37 smithi145 ceph-mon[26606]: pgmap v77: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 314 B/s wr, 0 op/s 2024-04-22T15:11:37.528 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:37 smithi145 ceph-mon[21899]: from='client.14706 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:37.528 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:37 smithi145 ceph-mon[21899]: Saving service iscsi.foo spec with placement count:1 2024-04-22T15:11:37.528 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:37 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:37.529 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:37 smithi145 ceph-mon[21899]: pgmap v77: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 314 B/s wr, 0 op/s 2024-04-22T15:11:37.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:37 smithi172 ceph-mon[22747]: from='client.14706 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:11:37.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:37 smithi172 ceph-mon[22747]: Saving service iscsi.foo spec with placement count:1 2024-04-22T15:11:37.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:37 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:37.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:37 smithi172 ceph-mon[22747]: pgmap v77: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 314 B/s wr, 0 op/s 2024-04-22T15:11:38.218 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:11:37 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:11:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:11:38.525 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:38 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:38.525 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:38 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.glxwwa", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T15:11:38.525 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:38 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.glxwwa", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-22T15:11:38.525 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:38 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:38.525 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:38 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:38.525 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:38 smithi145 ceph-mon[21899]: Deploying daemon rgw.foo.smithi172.glxwwa on smithi172 2024-04-22T15:11:38.526 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:38 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:38.526 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:38 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.glxwwa", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T15:11:38.526 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:38 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.glxwwa", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-22T15:11:38.526 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:38 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:38.526 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:38 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:38.526 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:38 smithi145 ceph-mon[26606]: Deploying daemon rgw.foo.smithi172.glxwwa on smithi172 2024-04-22T15:11:38.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:38 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:38.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:38 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.glxwwa", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T15:11:38.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:38 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.glxwwa", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-22T15:11:38.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:38 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:38.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:38 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:38.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:38 smithi172 ceph-mon[22747]: Deploying daemon rgw.foo.smithi172.glxwwa on smithi172 2024-04-22T15:11:40.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:40 smithi145 ceph-mon[21899]: pgmap v78: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-22T15:11:40.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:40 smithi145 ceph-mon[21899]: osdmap e78: 8 total, 8 up, 8 in 2024-04-22T15:11:40.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:40 smithi145 ceph-mon[26606]: pgmap v78: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-22T15:11:40.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:40 smithi145 ceph-mon[26606]: osdmap e78: 8 total, 8 up, 8 in 2024-04-22T15:11:40.441 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:40 smithi172 ceph-mon[22747]: pgmap v78: 161 pgs: 161 active+clean; 581 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-22T15:11:40.441 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:40 smithi172 ceph-mon[22747]: osdmap e78: 8 total, 8 up, 8 in 2024-04-22T15:11:42.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:11:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:11:41] "GET /metrics HTTP/1.1" 200 197890 "" "Prometheus/2.33.4" 2024-04-22T15:11:42.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:42 smithi145 ceph-mon[21899]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 4.1 KiB/s wr, 132 op/s 2024-04-22T15:11:42.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:42 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:42.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:42 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:11:42.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:42 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:42.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:42 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:11:42.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:42 smithi145 ceph-mon[26606]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 4.1 KiB/s wr, 132 op/s 2024-04-22T15:11:42.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:42 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:42.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:42 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:11:42.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:42 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:42.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:42 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:11:42.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:42 smithi172 ceph-mon[22747]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 4.1 KiB/s wr, 132 op/s 2024-04-22T15:11:42.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:42 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:42.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:42 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:11:42.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:42 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:42.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:42 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:11:43.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 2]}]: dispatch 2024-04-22T15:11:43.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-04-22T15:11:43.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-04-22T15:11:43.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2024-04-22T15:11:43.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 2]}]: dispatch 2024-04-22T15:11:43.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-04-22T15:11:43.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-04-22T15:11:43.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2024-04-22T15:11:43.308 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 2]}]: dispatch 2024-04-22T15:11:43.308 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-04-22T15:11:43.309 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-04-22T15:11:43.309 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2024-04-22T15:11:44.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:44 smithi145 ceph-mon[26606]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 3.5 KiB/s wr, 112 op/s 2024-04-22T15:11:44.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:44 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 2]}]': finished 2024-04-22T15:11:44.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:44 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-04-22T15:11:44.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:44 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-04-22T15:11:44.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:44 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2024-04-22T15:11:44.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:44 smithi145 ceph-mon[26606]: osdmap e79: 8 total, 8 up, 8 in 2024-04-22T15:11:44.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:44 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:11:44.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:44 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:11:44.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:44 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:44.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:44 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:44.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:44 smithi145 ceph-mon[21899]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 3.5 KiB/s wr, 112 op/s 2024-04-22T15:11:44.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:44 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 2]}]': finished 2024-04-22T15:11:44.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:44 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-04-22T15:11:44.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:44 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-04-22T15:11:44.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:44 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2024-04-22T15:11:44.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:44 smithi145 ceph-mon[21899]: osdmap e79: 8 total, 8 up, 8 in 2024-04-22T15:11:44.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:44 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:11:44.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:44 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:11:44.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:44 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:44.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:44 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:44.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:44 smithi172 ceph-mon[22747]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 3.5 KiB/s wr, 112 op/s 2024-04-22T15:11:44.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:44 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 2]}]': finished 2024-04-22T15:11:44.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:44 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-04-22T15:11:44.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:44 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-04-22T15:11:44.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:44 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2024-04-22T15:11:44.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:44 smithi172 ceph-mon[22747]: osdmap e79: 8 total, 8 up, 8 in 2024-04-22T15:11:44.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:44 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:11:44.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:44 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:11:44.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:44 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:44.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:44 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:45.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:45 smithi145 ceph-mon[21899]: osdmap e80: 8 total, 8 up, 8 in 2024-04-22T15:11:45.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:45 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:45.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:45 smithi145 ceph-mon[21899]: Checking dashboard <-> RGW credentials 2024-04-22T15:11:45.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:45 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:45.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:45 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:45.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:45 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:45.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:45 smithi145 ceph-mon[21899]: Checking pool "foo" exists for service iscsi.foo 2024-04-22T15:11:45.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:45 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:11:45.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:45 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-04-22T15:11:45.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:45 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:45.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:45 smithi145 ceph-mon[21899]: Deploying daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:11:45.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:45 smithi145 ceph-mon[21899]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 185 KiB/s rd, 5.7 KiB/s wr, 353 op/s 2024-04-22T15:11:45.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:45 smithi145 ceph-mon[26606]: osdmap e80: 8 total, 8 up, 8 in 2024-04-22T15:11:45.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:45 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:45.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:45 smithi145 ceph-mon[26606]: Checking dashboard <-> RGW credentials 2024-04-22T15:11:45.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:45 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:45.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:45 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:45.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:45 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:45.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:45 smithi145 ceph-mon[26606]: Checking pool "foo" exists for service iscsi.foo 2024-04-22T15:11:45.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:45 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:11:45.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:45 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-04-22T15:11:45.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:45 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:45.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:45 smithi145 ceph-mon[26606]: Deploying daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:11:45.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:45 smithi145 ceph-mon[26606]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 185 KiB/s rd, 5.7 KiB/s wr, 353 op/s 2024-04-22T15:11:45.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:45 smithi172 ceph-mon[22747]: osdmap e80: 8 total, 8 up, 8 in 2024-04-22T15:11:45.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:45 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:45.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:45 smithi172 ceph-mon[22747]: Checking dashboard <-> RGW credentials 2024-04-22T15:11:45.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:45 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:45.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:45 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:45.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:45 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:45.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:45 smithi172 ceph-mon[22747]: Checking pool "foo" exists for service iscsi.foo 2024-04-22T15:11:45.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:45 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:11:45.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:45 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-04-22T15:11:45.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:45 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:45.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:45 smithi172 ceph-mon[22747]: Deploying daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:11:45.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:45 smithi172 ceph-mon[22747]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 185 KiB/s rd, 5.7 KiB/s wr, 353 op/s 2024-04-22T15:11:47.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:46 smithi145 ceph-mon[21899]: mgrmap e20: y(active, since 2m), standbys: x 2024-04-22T15:11:47.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:46 smithi145 ceph-mon[26606]: mgrmap e20: y(active, since 2m), standbys: x 2024-04-22T15:11:47.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:46 smithi172 ceph-mon[22747]: mgrmap e20: y(active, since 2m), standbys: x 2024-04-22T15:11:48.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:47 smithi145 ceph-mon[26606]: pgmap v85: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 81 KiB/s rd, 128 B/s wr, 125 op/s 2024-04-22T15:11:48.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:47 smithi145 ceph-mon[21899]: pgmap v85: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 81 KiB/s rd, 128 B/s wr, 125 op/s 2024-04-22T15:11:48.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:47 smithi172 ceph-mon[22747]: pgmap v85: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 81 KiB/s rd, 128 B/s wr, 125 op/s 2024-04-22T15:11:48.218 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:11:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:11:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:11:49.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:48 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:49.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:48 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:11:49.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:48 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:49.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:48 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:11:49.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:48 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:49.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:48 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/1903662047' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T15:11:49.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:48 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/1388395725' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/3974605882"}]: dispatch 2024-04-22T15:11:49.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:48 smithi172 ceph-mon[22747]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/3974605882"}]: dispatch 2024-04-22T15:11:49.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:48 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:49.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:48 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:11:49.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:48 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:49.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:48 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:11:49.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:48 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:49.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:48 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/1903662047' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T15:11:49.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:48 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/1388395725' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/3974605882"}]: dispatch 2024-04-22T15:11:49.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:48 smithi145 ceph-mon[26606]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/3974605882"}]: dispatch 2024-04-22T15:11:49.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:48 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:49.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:48 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:11:49.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:48 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:49.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:48 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:11:49.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:48 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:49.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:48 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/1903662047' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T15:11:49.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:48 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/1388395725' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/3974605882"}]: dispatch 2024-04-22T15:11:49.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:48 smithi145 ceph-mon[21899]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/3974605882"}]: dispatch 2024-04-22T15:11:50.326 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:50 smithi145 ceph-mon[21899]: pgmap v86: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 81 KiB/s rd, 127 B/s wr, 124 op/s 2024-04-22T15:11:50.326 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:50 smithi145 ceph-mon[21899]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/3974605882"}]': finished 2024-04-22T15:11:50.326 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:50 smithi145 ceph-mon[21899]: osdmap e81: 8 total, 8 up, 8 in 2024-04-22T15:11:50.327 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:50 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/4283940571' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3245087794"}]: dispatch 2024-04-22T15:11:50.327 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:50 smithi145 ceph-mon[26606]: pgmap v86: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 81 KiB/s rd, 127 B/s wr, 124 op/s 2024-04-22T15:11:50.327 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:50 smithi145 ceph-mon[26606]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/3974605882"}]': finished 2024-04-22T15:11:50.327 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:50 smithi145 ceph-mon[26606]: osdmap e81: 8 total, 8 up, 8 in 2024-04-22T15:11:50.327 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:50 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/4283940571' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3245087794"}]: dispatch 2024-04-22T15:11:50.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:50 smithi172 ceph-mon[22747]: pgmap v86: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 81 KiB/s rd, 127 B/s wr, 124 op/s 2024-04-22T15:11:50.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:50 smithi172 ceph-mon[22747]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/3974605882"}]': finished 2024-04-22T15:11:50.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:50 smithi172 ceph-mon[22747]: osdmap e81: 8 total, 8 up, 8 in 2024-04-22T15:11:50.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:50 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/4283940571' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3245087794"}]: dispatch 2024-04-22T15:11:51.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:51 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/4283940571' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3245087794"}]': finished 2024-04-22T15:11:51.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:51 smithi145 ceph-mon[26606]: osdmap e82: 8 total, 8 up, 8 in 2024-04-22T15:11:51.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:51 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3055752770' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2226900857"}]: dispatch 2024-04-22T15:11:51.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:51 smithi145 ceph-mon[26606]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2226900857"}]: dispatch 2024-04-22T15:11:51.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:51 smithi145 ceph-mon[26606]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 6.2 KiB/s rd, 442 B/s wr, 7 op/s; 13 B/s, 0 objects/s recovering 2024-04-22T15:11:51.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:51 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/4283940571' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3245087794"}]': finished 2024-04-22T15:11:51.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:51 smithi145 ceph-mon[21899]: osdmap e82: 8 total, 8 up, 8 in 2024-04-22T15:11:51.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:51 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3055752770' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2226900857"}]: dispatch 2024-04-22T15:11:51.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:51 smithi145 ceph-mon[21899]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2226900857"}]: dispatch 2024-04-22T15:11:51.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:51 smithi145 ceph-mon[21899]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 6.2 KiB/s rd, 442 B/s wr, 7 op/s; 13 B/s, 0 objects/s recovering 2024-04-22T15:11:51.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:51 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/4283940571' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3245087794"}]': finished 2024-04-22T15:11:51.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:51 smithi172 ceph-mon[22747]: osdmap e82: 8 total, 8 up, 8 in 2024-04-22T15:11:51.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:51 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/3055752770' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2226900857"}]: dispatch 2024-04-22T15:11:51.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:51 smithi172 ceph-mon[22747]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2226900857"}]: dispatch 2024-04-22T15:11:51.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:51 smithi172 ceph-mon[22747]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 6.2 KiB/s rd, 442 B/s wr, 7 op/s; 13 B/s, 0 objects/s recovering 2024-04-22T15:11:52.091 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:11:51 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:11:51] "GET /metrics HTTP/1.1" 200 207181 "" "Prometheus/2.33.4" 2024-04-22T15:11:52.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[26606]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2226900857"}]': finished 2024-04-22T15:11:52.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[26606]: osdmap e83: 8 total, 8 up, 8 in 2024-04-22T15:11:52.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:52.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/1874341256' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2747862617"}]: dispatch 2024-04-22T15:11:52.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:52.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:52.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T15:11:52.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[26606]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T15:11:52.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[26606]: Adding iSCSI gateway http://:@172.21.15.145:5000 to Dashboard 2024-04-22T15:11:52.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T15:11:52.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[26606]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T15:11:52.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:52.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi145"}]: dispatch 2024-04-22T15:11:52.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[26606]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi145"}]: dispatch 2024-04-22T15:11:52.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:52.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:11:52.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:52.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:11:52.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[26606]: Checking dashboard <-> RGW credentials 2024-04-22T15:11:52.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:52.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[21899]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2226900857"}]': finished 2024-04-22T15:11:52.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[21899]: osdmap e83: 8 total, 8 up, 8 in 2024-04-22T15:11:52.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:52.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/1874341256' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2747862617"}]: dispatch 2024-04-22T15:11:52.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:52.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:52.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T15:11:52.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[21899]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T15:11:52.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[21899]: Adding iSCSI gateway http://:@172.21.15.145:5000 to Dashboard 2024-04-22T15:11:52.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T15:11:52.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[21899]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T15:11:52.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:52.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi145"}]: dispatch 2024-04-22T15:11:52.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[21899]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi145"}]: dispatch 2024-04-22T15:11:52.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:52.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:11:52.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:52.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:11:52.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[21899]: Checking dashboard <-> RGW credentials 2024-04-22T15:11:52.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:52 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:52.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:52 smithi172 ceph-mon[22747]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2226900857"}]': finished 2024-04-22T15:11:52.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:52 smithi172 ceph-mon[22747]: osdmap e83: 8 total, 8 up, 8 in 2024-04-22T15:11:52.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:52 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:52.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:52 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/1874341256' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2747862617"}]: dispatch 2024-04-22T15:11:52.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:52 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:52.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:52 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:52.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:52 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T15:11:52.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:52 smithi172 ceph-mon[22747]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T15:11:52.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:52 smithi172 ceph-mon[22747]: Adding iSCSI gateway http://:@172.21.15.145:5000 to Dashboard 2024-04-22T15:11:52.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:52 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T15:11:52.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:52 smithi172 ceph-mon[22747]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T15:11:52.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:52 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:52.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:52 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi145"}]: dispatch 2024-04-22T15:11:52.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:52 smithi172 ceph-mon[22747]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi145"}]: dispatch 2024-04-22T15:11:52.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:52 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:52.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:52 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:11:52.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:52 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:11:52.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:52 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:11:52.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:52 smithi172 ceph-mon[22747]: Checking dashboard <-> RGW credentials 2024-04-22T15:11:52.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:52 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:11:53.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:53 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/1874341256' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2747862617"}]': finished 2024-04-22T15:11:53.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:53 smithi172 ceph-mon[22747]: osdmap e84: 8 total, 8 up, 8 in 2024-04-22T15:11:53.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:53 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/1612395605' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/3974605882"}]: dispatch 2024-04-22T15:11:53.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:53 smithi172 ceph-mon[22747]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/3974605882"}]: dispatch 2024-04-22T15:11:53.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:53 smithi172 ceph-mon[22747]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 767 B/s wr, 13 op/s; 22 B/s, 0 objects/s recovering 2024-04-22T15:11:53.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:53 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/1874341256' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2747862617"}]': finished 2024-04-22T15:11:53.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:53 smithi145 ceph-mon[21899]: osdmap e84: 8 total, 8 up, 8 in 2024-04-22T15:11:53.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:53 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/1612395605' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/3974605882"}]: dispatch 2024-04-22T15:11:53.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:53 smithi145 ceph-mon[21899]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/3974605882"}]: dispatch 2024-04-22T15:11:53.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:53 smithi145 ceph-mon[21899]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 767 B/s wr, 13 op/s; 22 B/s, 0 objects/s recovering 2024-04-22T15:11:53.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:53 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/1874341256' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2747862617"}]': finished 2024-04-22T15:11:53.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:53 smithi145 ceph-mon[26606]: osdmap e84: 8 total, 8 up, 8 in 2024-04-22T15:11:53.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:53 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/1612395605' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/3974605882"}]: dispatch 2024-04-22T15:11:53.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:53 smithi145 ceph-mon[26606]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/3974605882"}]: dispatch 2024-04-22T15:11:53.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:53 smithi145 ceph-mon[26606]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 767 B/s wr, 13 op/s; 22 B/s, 0 objects/s recovering 2024-04-22T15:11:54.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:54 smithi172 ceph-mon[22747]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/3974605882"}]': finished 2024-04-22T15:11:54.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:54 smithi172 ceph-mon[22747]: osdmap e85: 8 total, 8 up, 8 in 2024-04-22T15:11:54.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:54 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/3595211304' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/1271607645"}]: dispatch 2024-04-22T15:11:54.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:54 smithi172 ceph-mon[22747]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/1271607645"}]: dispatch 2024-04-22T15:11:54.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:54 smithi145 ceph-mon[21899]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/3974605882"}]': finished 2024-04-22T15:11:54.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:54 smithi145 ceph-mon[21899]: osdmap e85: 8 total, 8 up, 8 in 2024-04-22T15:11:54.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:54 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3595211304' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/1271607645"}]: dispatch 2024-04-22T15:11:54.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:54 smithi145 ceph-mon[21899]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/1271607645"}]: dispatch 2024-04-22T15:11:54.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:54 smithi145 ceph-mon[26606]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/3974605882"}]': finished 2024-04-22T15:11:54.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:54 smithi145 ceph-mon[26606]: osdmap e85: 8 total, 8 up, 8 in 2024-04-22T15:11:54.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:54 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3595211304' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/1271607645"}]: dispatch 2024-04-22T15:11:54.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:54 smithi145 ceph-mon[26606]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/1271607645"}]: dispatch 2024-04-22T15:11:55.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:55 smithi172 ceph-mon[22747]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/1271607645"}]': finished 2024-04-22T15:11:55.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:55 smithi172 ceph-mon[22747]: osdmap e86: 8 total, 8 up, 8 in 2024-04-22T15:11:55.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:55 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/1673991330' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1075985196"}]: dispatch 2024-04-22T15:11:55.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:55 smithi172 ceph-mon[22747]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-04-22T15:11:55.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:55 smithi145 ceph-mon[21899]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/1271607645"}]': finished 2024-04-22T15:11:55.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:55 smithi145 ceph-mon[21899]: osdmap e86: 8 total, 8 up, 8 in 2024-04-22T15:11:55.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:55 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/1673991330' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1075985196"}]: dispatch 2024-04-22T15:11:55.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:55 smithi145 ceph-mon[21899]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-04-22T15:11:55.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:55 smithi145 ceph-mon[26606]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/1271607645"}]': finished 2024-04-22T15:11:55.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:55 smithi145 ceph-mon[26606]: osdmap e86: 8 total, 8 up, 8 in 2024-04-22T15:11:55.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:55 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/1673991330' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1075985196"}]: dispatch 2024-04-22T15:11:55.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:55 smithi145 ceph-mon[26606]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-04-22T15:11:56.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:56 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/1673991330' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1075985196"}]': finished 2024-04-22T15:11:56.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:56 smithi145 ceph-mon[21899]: osdmap e87: 8 total, 8 up, 8 in 2024-04-22T15:11:56.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:56 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3364220235' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3793806840"}]: dispatch 2024-04-22T15:11:56.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:56 smithi145 ceph-mon[21899]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3793806840"}]: dispatch 2024-04-22T15:11:56.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:56 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/1673991330' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1075985196"}]': finished 2024-04-22T15:11:56.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:56 smithi145 ceph-mon[26606]: osdmap e87: 8 total, 8 up, 8 in 2024-04-22T15:11:56.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:56 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3364220235' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3793806840"}]: dispatch 2024-04-22T15:11:56.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:56 smithi145 ceph-mon[26606]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3793806840"}]: dispatch 2024-04-22T15:11:56.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:56 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/1673991330' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1075985196"}]': finished 2024-04-22T15:11:56.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:56 smithi172 ceph-mon[22747]: osdmap e87: 8 total, 8 up, 8 in 2024-04-22T15:11:56.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:56 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/3364220235' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3793806840"}]: dispatch 2024-04-22T15:11:56.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:56 smithi172 ceph-mon[22747]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3793806840"}]: dispatch 2024-04-22T15:11:57.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:57 smithi145 ceph-mon[21899]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3793806840"}]': finished 2024-04-22T15:11:57.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:57 smithi145 ceph-mon[21899]: osdmap e88: 8 total, 8 up, 8 in 2024-04-22T15:11:57.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:57 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2537747068' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1268350667"}]: dispatch 2024-04-22T15:11:57.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:57 smithi145 ceph-mon[21899]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1268350667"}]: dispatch 2024-04-22T15:11:57.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:57 smithi145 ceph-mon[21899]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-04-22T15:11:57.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:57 smithi145 ceph-mon[26606]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3793806840"}]': finished 2024-04-22T15:11:57.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:57 smithi145 ceph-mon[26606]: osdmap e88: 8 total, 8 up, 8 in 2024-04-22T15:11:57.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:57 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2537747068' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1268350667"}]: dispatch 2024-04-22T15:11:57.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:57 smithi145 ceph-mon[26606]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1268350667"}]: dispatch 2024-04-22T15:11:57.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:57 smithi145 ceph-mon[26606]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-04-22T15:11:57.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:57 smithi172 ceph-mon[22747]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3793806840"}]': finished 2024-04-22T15:11:57.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:57 smithi172 ceph-mon[22747]: osdmap e88: 8 total, 8 up, 8 in 2024-04-22T15:11:57.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:57 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2537747068' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1268350667"}]: dispatch 2024-04-22T15:11:57.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:57 smithi172 ceph-mon[22747]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1268350667"}]: dispatch 2024-04-22T15:11:57.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:57 smithi172 ceph-mon[22747]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-04-22T15:11:58.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:58 smithi145 ceph-mon[21899]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1268350667"}]': finished 2024-04-22T15:11:58.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:58 smithi145 ceph-mon[21899]: osdmap e89: 8 total, 8 up, 8 in 2024-04-22T15:11:58.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:58 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3093750482' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1261892263"}]: dispatch 2024-04-22T15:11:58.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:58 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:11:58.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:58 smithi145 ceph-mon[26606]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1268350667"}]': finished 2024-04-22T15:11:58.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:58 smithi145 ceph-mon[26606]: osdmap e89: 8 total, 8 up, 8 in 2024-04-22T15:11:58.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:58 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3093750482' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1261892263"}]: dispatch 2024-04-22T15:11:58.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:58 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:11:58.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:11:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:11:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:11:58.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:58 smithi172 ceph-mon[22747]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1268350667"}]': finished 2024-04-22T15:11:58.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:58 smithi172 ceph-mon[22747]: osdmap e89: 8 total, 8 up, 8 in 2024-04-22T15:11:58.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:58 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/3093750482' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1261892263"}]: dispatch 2024-04-22T15:11:58.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:58 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:11:59.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:59 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3093750482' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1261892263"}]': finished 2024-04-22T15:11:59.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:59 smithi145 ceph-mon[26606]: osdmap e90: 8 total, 8 up, 8 in 2024-04-22T15:11:59.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:59 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/4107335394' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1333376048"}]: dispatch 2024-04-22T15:11:59.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:59 smithi145 ceph-mon[26606]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1333376048"}]: dispatch 2024-04-22T15:11:59.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:11:59 smithi145 ceph-mon[26606]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:59.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:59 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3093750482' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1261892263"}]': finished 2024-04-22T15:11:59.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:59 smithi145 ceph-mon[21899]: osdmap e90: 8 total, 8 up, 8 in 2024-04-22T15:11:59.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:59 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/4107335394' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1333376048"}]: dispatch 2024-04-22T15:11:59.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:59 smithi145 ceph-mon[21899]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1333376048"}]: dispatch 2024-04-22T15:11:59.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:11:59 smithi145 ceph-mon[21899]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:11:59.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:59 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/3093750482' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1261892263"}]': finished 2024-04-22T15:11:59.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:59 smithi172 ceph-mon[22747]: osdmap e90: 8 total, 8 up, 8 in 2024-04-22T15:11:59.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:59 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/4107335394' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1333376048"}]: dispatch 2024-04-22T15:11:59.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:59 smithi172 ceph-mon[22747]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1333376048"}]: dispatch 2024-04-22T15:11:59.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:11:59 smithi172 ceph-mon[22747]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:12:00.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:00 smithi172 ceph-mon[22747]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1333376048"}]': finished 2024-04-22T15:12:00.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:00 smithi172 ceph-mon[22747]: osdmap e91: 8 total, 8 up, 8 in 2024-04-22T15:12:00.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:00 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2885446650' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/4174602953"}]: dispatch 2024-04-22T15:12:00.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:00 smithi172 ceph-mon[22747]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/4174602953"}]: dispatch 2024-04-22T15:12:00.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:00 smithi145 ceph-mon[21899]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1333376048"}]': finished 2024-04-22T15:12:00.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:00 smithi145 ceph-mon[21899]: osdmap e91: 8 total, 8 up, 8 in 2024-04-22T15:12:00.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:00 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2885446650' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/4174602953"}]: dispatch 2024-04-22T15:12:00.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:00 smithi145 ceph-mon[21899]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/4174602953"}]: dispatch 2024-04-22T15:12:00.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:00 smithi145 ceph-mon[26606]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1333376048"}]': finished 2024-04-22T15:12:00.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:00 smithi145 ceph-mon[26606]: osdmap e91: 8 total, 8 up, 8 in 2024-04-22T15:12:00.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:00 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2885446650' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/4174602953"}]: dispatch 2024-04-22T15:12:00.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:00 smithi145 ceph-mon[26606]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/4174602953"}]: dispatch 2024-04-22T15:12:01.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:01 smithi172 ceph-mon[22747]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/4174602953"}]': finished 2024-04-22T15:12:01.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:01 smithi172 ceph-mon[22747]: osdmap e92: 8 total, 8 up, 8 in 2024-04-22T15:12:01.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:01 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/4978574' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2791239557"}]: dispatch 2024-04-22T15:12:01.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:01 smithi172 ceph-mon[22747]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2791239557"}]: dispatch 2024-04-22T15:12:01.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:01 smithi172 ceph-mon[22747]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:01.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:01 smithi145 ceph-mon[26606]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/4174602953"}]': finished 2024-04-22T15:12:01.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:01 smithi145 ceph-mon[26606]: osdmap e92: 8 total, 8 up, 8 in 2024-04-22T15:12:01.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:01 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/4978574' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2791239557"}]: dispatch 2024-04-22T15:12:01.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:01 smithi145 ceph-mon[26606]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2791239557"}]: dispatch 2024-04-22T15:12:01.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:01 smithi145 ceph-mon[26606]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:01.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:01 smithi145 ceph-mon[21899]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/4174602953"}]': finished 2024-04-22T15:12:01.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:01 smithi145 ceph-mon[21899]: osdmap e92: 8 total, 8 up, 8 in 2024-04-22T15:12:01.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:01 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/4978574' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2791239557"}]: dispatch 2024-04-22T15:12:01.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:01 smithi145 ceph-mon[21899]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2791239557"}]: dispatch 2024-04-22T15:12:01.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:01 smithi145 ceph-mon[21899]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:02.263 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:12:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:12:01] "GET /metrics HTTP/1.1" 200 208094 "" "Prometheus/2.33.4" 2024-04-22T15:12:02.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:02 smithi145 ceph-mon[21899]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2791239557"}]': finished 2024-04-22T15:12:02.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:02 smithi145 ceph-mon[21899]: osdmap e93: 8 total, 8 up, 8 in 2024-04-22T15:12:02.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:02 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/405809105' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/218026216"}]: dispatch 2024-04-22T15:12:02.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:02 smithi145 ceph-mon[26606]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2791239557"}]': finished 2024-04-22T15:12:02.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:02 smithi145 ceph-mon[26606]: osdmap e93: 8 total, 8 up, 8 in 2024-04-22T15:12:02.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:02 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/405809105' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/218026216"}]: dispatch 2024-04-22T15:12:02.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:02 smithi172 ceph-mon[22747]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2791239557"}]': finished 2024-04-22T15:12:02.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:02 smithi172 ceph-mon[22747]: osdmap e93: 8 total, 8 up, 8 in 2024-04-22T15:12:02.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:02 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/405809105' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/218026216"}]: dispatch 2024-04-22T15:12:03.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:03 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/405809105' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/218026216"}]': finished 2024-04-22T15:12:03.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:03 smithi145 ceph-mon[21899]: osdmap e94: 8 total, 8 up, 8 in 2024-04-22T15:12:03.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:03 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/1296774045' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/1271607645"}]: dispatch 2024-04-22T15:12:03.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:03 smithi145 ceph-mon[21899]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/1271607645"}]: dispatch 2024-04-22T15:12:03.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:03 smithi145 ceph-mon[21899]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:03.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:03 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/405809105' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/218026216"}]': finished 2024-04-22T15:12:03.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:03 smithi145 ceph-mon[26606]: osdmap e94: 8 total, 8 up, 8 in 2024-04-22T15:12:03.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:03 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/1296774045' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/1271607645"}]: dispatch 2024-04-22T15:12:03.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:03 smithi145 ceph-mon[26606]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/1271607645"}]: dispatch 2024-04-22T15:12:03.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:03 smithi145 ceph-mon[26606]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:03.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:03 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/405809105' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/218026216"}]': finished 2024-04-22T15:12:03.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:03 smithi172 ceph-mon[22747]: osdmap e94: 8 total, 8 up, 8 in 2024-04-22T15:12:03.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:03 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/1296774045' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/1271607645"}]: dispatch 2024-04-22T15:12:03.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:03 smithi172 ceph-mon[22747]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/1271607645"}]: dispatch 2024-04-22T15:12:03.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:03 smithi172 ceph-mon[22747]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:04.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:04 smithi145 ceph-mon[21899]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/1271607645"}]': finished 2024-04-22T15:12:04.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:04 smithi145 ceph-mon[21899]: osdmap e95: 8 total, 8 up, 8 in 2024-04-22T15:12:04.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:04 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2416741449' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/4174602953"}]: dispatch 2024-04-22T15:12:04.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:04 smithi145 ceph-mon[26606]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/1271607645"}]': finished 2024-04-22T15:12:04.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:04 smithi145 ceph-mon[26606]: osdmap e95: 8 total, 8 up, 8 in 2024-04-22T15:12:04.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:04 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2416741449' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/4174602953"}]: dispatch 2024-04-22T15:12:04.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:04 smithi172 ceph-mon[22747]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/1271607645"}]': finished 2024-04-22T15:12:04.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:04 smithi172 ceph-mon[22747]: osdmap e95: 8 total, 8 up, 8 in 2024-04-22T15:12:04.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:04 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2416741449' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/4174602953"}]: dispatch 2024-04-22T15:12:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:05 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2416741449' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/4174602953"}]': finished 2024-04-22T15:12:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:05 smithi145 ceph-mon[26606]: osdmap e96: 8 total, 8 up, 8 in 2024-04-22T15:12:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:05 smithi145 ceph-mon[26606]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:05.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:05 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2416741449' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/4174602953"}]': finished 2024-04-22T15:12:05.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:05 smithi145 ceph-mon[21899]: osdmap e96: 8 total, 8 up, 8 in 2024-04-22T15:12:05.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:05 smithi145 ceph-mon[21899]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:05.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:05 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2416741449' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/4174602953"}]': finished 2024-04-22T15:12:05.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:05 smithi172 ceph-mon[22747]: osdmap e96: 8 total, 8 up, 8 in 2024-04-22T15:12:05.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:05 smithi172 ceph-mon[22747]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:08.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:08 smithi145 ceph-mon[21899]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 892 B/s rd, 0 op/s 2024-04-22T15:12:08.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:08 smithi145 ceph-mon[26606]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 892 B/s rd, 0 op/s 2024-04-22T15:12:08.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:08 smithi172 ceph-mon[22747]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 892 B/s rd, 0 op/s 2024-04-22T15:12:08.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:12:07 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:12:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:12:09.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:09 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:12:09.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:09 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:12:09.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:09 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:12:10.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:10 smithi145 ceph-mon[21899]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-04-22T15:12:10.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:10 smithi145 ceph-mon[26606]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-04-22T15:12:10.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:10 smithi172 ceph-mon[22747]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-04-22T15:12:11.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:11 smithi145 ceph-mon[21899]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:11.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:11 smithi145 ceph-mon[26606]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:11.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:11 smithi172 ceph-mon[22747]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:12.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:12:11 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:12:11] "GET /metrics HTTP/1.1" 200 208094 "" "Prometheus/2.33.4" 2024-04-22T15:12:14.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:14 smithi145 ceph-mon[21899]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:12:14.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:14 smithi145 ceph-mon[26606]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:12:14.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:14 smithi172 ceph-mon[22747]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:12:15.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:15 smithi145 ceph-mon[21899]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 955 B/s rd, 0 op/s 2024-04-22T15:12:15.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:15 smithi145 ceph-mon[26606]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 955 B/s rd, 0 op/s 2024-04-22T15:12:15.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:15 smithi172 ceph-mon[22747]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 955 B/s rd, 0 op/s 2024-04-22T15:12:18.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:18 smithi145 ceph-mon[26606]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:18.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:18 smithi145 ceph-mon[21899]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:18.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:12:17 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:12:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:12:18.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:18 smithi172 ceph-mon[22747]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:19.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:19 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:12:19.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:19 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:12:19.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:19 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:12:20.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:20 smithi145 ceph-mon[26606]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:20.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:20 smithi145 ceph-mon[21899]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:20.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:20 smithi172 ceph-mon[22747]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:21.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:21 smithi145 ceph-mon[26606]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:21.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:21 smithi145 ceph-mon[21899]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:21.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:21 smithi172 ceph-mon[22747]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:22.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:12:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:12:21] "GET /metrics HTTP/1.1" 200 208097 "" "Prometheus/2.33.4" 2024-04-22T15:12:24.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:24 smithi145 ceph-mon[21899]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:24.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:24 smithi145 ceph-mon[26606]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:24.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:24 smithi172 ceph-mon[22747]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:25.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:25 smithi145 ceph-mon[21899]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:25.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:25 smithi145 ceph-mon[26606]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:25.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:25 smithi172 ceph-mon[22747]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:27.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:27 smithi145 ceph-mon[21899]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:27.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:27 smithi145 ceph-mon[26606]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:27.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:27 smithi172 ceph-mon[22747]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:28.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:12:27 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:12:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:12:30.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:30 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:12:30.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:30 smithi145 ceph-mon[26606]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:30.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:30 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:12:30.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:30 smithi145 ceph-mon[21899]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:30.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:30 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:12:30.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:30 smithi172 ceph-mon[22747]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:32.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:32 smithi145 ceph-mon[21899]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:32.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:32 smithi145 ceph-mon[26606]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:32.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:12:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:12:31] "GET /metrics HTTP/1.1" 200 208055 "" "Prometheus/2.33.4" 2024-04-22T15:12:32.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:32 smithi172 ceph-mon[22747]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:34.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:34 smithi145 ceph-mon[21899]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:34.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:34 smithi145 ceph-mon[26606]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:34.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:34 smithi172 ceph-mon[22747]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:35.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:35 smithi145 ceph-mon[21899]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:35.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:35 smithi145 ceph-mon[26606]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:35.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:35 smithi172 ceph-mon[22747]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:37.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:37 smithi145 ceph-mon[21899]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:37.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:37 smithi145 ceph-mon[26606]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:37.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:37 smithi172 ceph-mon[22747]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:38.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:12:37 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:12:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:12:40.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:40 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:12:40.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:40 smithi145 ceph-mon[21899]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:40.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:40 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:12:40.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:40 smithi145 ceph-mon[26606]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:40.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:40 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:12:40.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:40 smithi172 ceph-mon[22747]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:42.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:42 smithi145 ceph-mon[21899]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:42.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:42 smithi145 ceph-mon[26606]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:42.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:12:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:12:41] "GET /metrics HTTP/1.1" 200 208055 "" "Prometheus/2.33.4" 2024-04-22T15:12:42.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:42 smithi172 ceph-mon[22747]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:43.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:43 smithi145 ceph-mon[26606]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:43.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:12:43.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:12:43.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:43 smithi145 ceph-mon[21899]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:43.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:12:43.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:12:43.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:43 smithi172 ceph-mon[22747]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:43.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:12:43.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:12:46.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:46 smithi145 ceph-mon[21899]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:46.323 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:46 smithi145 ceph-mon[26606]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:46.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:46 smithi172 ceph-mon[22747]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:48.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:48 smithi145 ceph-mon[21899]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:48.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:48 smithi145 ceph-mon[26606]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:48.467 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:12:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:12:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:12:48.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:48 smithi172 ceph-mon[22747]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:50.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:50 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:12:50.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:50 smithi145 ceph-mon[21899]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:50.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:50 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:12:50.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:50 smithi145 ceph-mon[26606]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:50.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:50 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:12:50.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:50 smithi172 ceph-mon[22747]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:52.248 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:12:51 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:12:51] "GET /metrics HTTP/1.1" 200 208005 "" "Prometheus/2.33.4" 2024-04-22T15:12:52.248 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:52 smithi145 ceph-mon[26606]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:52.248 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:52 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:12:52.248 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:52 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:12:52.248 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:52 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:12:52.249 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:52 smithi145 ceph-mon[21899]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:52.249 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:52 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:12:52.249 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:52 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:12:52.249 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:52 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:12:52.250 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:52 smithi172 ceph-mon[22747]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:52.250 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:52 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:12:52.250 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:52 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:12:52.250 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:52 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:12:53.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:53 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:12:53.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:53 smithi145 ceph-mon[21899]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:53.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:53 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:12:53.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:53 smithi145 ceph-mon[26606]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:53.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:53 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:12:53.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:53 smithi172 ceph-mon[22747]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:56.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:56 smithi145 ceph-mon[21899]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:56.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:56 smithi145 ceph-mon[26606]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:56.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:56 smithi172 ceph-mon[22747]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:12:57.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:12:57 smithi145 ceph-mon[21899]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:57.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:12:57 smithi145 ceph-mon[26606]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:57.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:12:57 smithi172 ceph-mon[22747]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:12:58.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:12:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:12:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:13:00.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:00 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:13:00.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:00 smithi145 ceph-mon[26606]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:00.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:00 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:13:00.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:00 smithi145 ceph-mon[21899]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:00.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:00 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:13:00.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:00 smithi172 ceph-mon[22747]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:02.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:02 smithi145 ceph-mon[21899]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:02.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:02 smithi145 ceph-mon[26606]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:02.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:13:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:13:01] "GET /metrics HTTP/1.1" 200 208001 "" "Prometheus/2.33.4" 2024-04-22T15:13:02.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:02 smithi172 ceph-mon[22747]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:03.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:03 smithi145 ceph-mon[21899]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:03.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:03 smithi145 ceph-mon[26606]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:03.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:03 smithi172 ceph-mon[22747]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:06.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:06 smithi145 ceph-mon[21899]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:06.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:06 smithi145 ceph-mon[26606]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:06.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:06 smithi172 ceph-mon[22747]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:07.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:07 smithi145 ceph-mon[21899]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:07.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:07 smithi145 ceph-mon[26606]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:07.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:07 smithi172 ceph-mon[22747]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:08.467 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:13:07 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:13:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:13:10.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:10 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:13:10.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:10 smithi145 ceph-mon[21899]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:10.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:10 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:13:10.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:10 smithi145 ceph-mon[26606]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:10.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:10 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:13:10.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:10 smithi172 ceph-mon[22747]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:12.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:12 smithi145 ceph-mon[21899]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:12.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:12 smithi145 ceph-mon[26606]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:12.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:13:11 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:13:11] "GET /metrics HTTP/1.1" 200 208001 "" "Prometheus/2.33.4" 2024-04-22T15:13:12.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:12 smithi172 ceph-mon[22747]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:13.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:13 smithi145 ceph-mon[21899]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:13.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:13 smithi145 ceph-mon[26606]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:13.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:13 smithi172 ceph-mon[22747]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:16.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:16 smithi145 ceph-mon[21899]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:16.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:16 smithi145 ceph-mon[26606]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:16.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:16 smithi172 ceph-mon[22747]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:17.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:17 smithi145 ceph-mon[21899]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:17.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:17 smithi145 ceph-mon[26606]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:17.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:17 smithi172 ceph-mon[22747]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:18.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:13:17 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:13:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:13:20.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:20 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:13:20.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:20 smithi145 ceph-mon[21899]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:20.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:20 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:13:20.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:20 smithi145 ceph-mon[26606]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:20.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:20 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:13:20.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:20 smithi172 ceph-mon[22747]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:21.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:21 smithi145 ceph-mon[21899]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:21.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:21 smithi145 ceph-mon[26606]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:21.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:21 smithi172 ceph-mon[22747]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:22.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:13:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:13:21] "GET /metrics HTTP/1.1" 200 208007 "" "Prometheus/2.33.4" 2024-04-22T15:13:23.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:23 smithi145 ceph-mon[26606]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:23.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:23 smithi145 ceph-mon[21899]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:23.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:23 smithi172 ceph-mon[22747]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:25.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:25 smithi145 ceph-mon[26606]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:25.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:25 smithi145 ceph-mon[21899]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:25.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:25 smithi172 ceph-mon[22747]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:27.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:27 smithi145 ceph-mon[26606]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:27.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:27 smithi145 ceph-mon[21899]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:27.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:27 smithi172 ceph-mon[22747]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:28.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:13:27 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:13:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:13:30.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:30 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:13:30.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:30 smithi145 ceph-mon[26606]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:30.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:30 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:13:30.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:30 smithi145 ceph-mon[21899]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:30.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:30 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:13:30.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:30 smithi172 ceph-mon[22747]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:31.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:31 smithi145 ceph-mon[21899]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:31.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:31 smithi145 ceph-mon[26606]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:31.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:31 smithi172 ceph-mon[22747]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:32.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:13:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:13:31] "GET /metrics HTTP/1.1" 200 208012 "" "Prometheus/2.33.4" 2024-04-22T15:13:33.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:33 smithi145 ceph-mon[21899]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:33.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:33 smithi145 ceph-mon[26606]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:33.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:33 smithi172 ceph-mon[22747]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:35.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:35 smithi145 ceph-mon[21899]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:35.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:35 smithi145 ceph-mon[26606]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:35.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:35 smithi172 ceph-mon[22747]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:37.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:37 smithi145 ceph-mon[21899]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:37.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:37 smithi145 ceph-mon[26606]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:37.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:37 smithi172 ceph-mon[22747]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:38.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:13:37 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:13:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:13:40.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:40 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:13:40.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:40 smithi145 ceph-mon[21899]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:40.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:40 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:13:40.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:40 smithi145 ceph-mon[26606]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:40.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:40 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:13:40.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:40 smithi172 ceph-mon[22747]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:41.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:41 smithi145 ceph-mon[21899]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:41.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:41 smithi145 ceph-mon[26606]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:41.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:41 smithi172 ceph-mon[22747]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:42.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:13:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:13:41] "GET /metrics HTTP/1.1" 200 208012 "" "Prometheus/2.33.4" 2024-04-22T15:13:43.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:43 smithi145 ceph-mon[21899]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:43.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:13:43.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:13:43.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:43 smithi145 ceph-mon[26606]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:43.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:13:43.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:13:43.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:43 smithi172 ceph-mon[22747]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:43.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:13:43.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:13:45.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:45 smithi145 ceph-mon[21899]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:45.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:45 smithi145 ceph-mon[26606]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:45.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:45 smithi172 ceph-mon[22747]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:47.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:47 smithi145 ceph-mon[26606]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T15:13:47.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:47 smithi145 ceph-mon[21899]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T15:13:47.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:47 smithi172 ceph-mon[22747]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T15:13:48.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:13:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:13:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:13:49.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:49 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:13:49.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:49 smithi145 ceph-mon[21899]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:49.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:49 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:13:49.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:49 smithi145 ceph-mon[26606]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:49.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:49 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:13:49.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:49 smithi172 ceph-mon[22747]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:51.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:51 smithi145 ceph-mon[26606]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:51.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:51 smithi145 ceph-mon[21899]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:51.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:51 smithi172 ceph-mon[22747]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:52.292 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:13:51 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:13:51] "GET /metrics HTTP/1.1" 200 208007 "" "Prometheus/2.33.4" 2024-04-22T15:13:52.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:52 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:13:52.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:52 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:13:52.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:52 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:13:52.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:52 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:13:52.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:52 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:13:52.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:52 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:13:52.578 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:52 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:13:52.578 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:52 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:13:52.578 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:52 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:13:53.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:53 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:13:53.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:53 smithi145 ceph-mon[21899]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:53.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:53 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:13:53.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:53 smithi145 ceph-mon[26606]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:53.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:53 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:13:53.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:53 smithi172 ceph-mon[22747]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:55.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:55 smithi145 ceph-mon[21899]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:55.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:55 smithi145 ceph-mon[26606]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:55.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:55 smithi172 ceph-mon[22747]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:57.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:57 smithi145 ceph-mon[21899]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:57.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:57 smithi145 ceph-mon[26606]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:57.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:57 smithi172 ceph-mon[22747]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:13:58.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:13:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:13:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:13:59.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:59 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:13:59.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:13:59 smithi145 ceph-mon[21899]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:59.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:59 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:13:59.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:13:59 smithi145 ceph-mon[26606]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:13:59.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:59 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:13:59.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:13:59 smithi172 ceph-mon[22747]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:01.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:01 smithi145 ceph-mon[21899]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:01.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:01 smithi145 ceph-mon[26606]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:01.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:01 smithi172 ceph-mon[22747]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:02.372 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:14:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:14:01] "GET /metrics HTTP/1.1" 200 208000 "" "Prometheus/2.33.4" 2024-04-22T15:14:03.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:03 smithi145 ceph-mon[21899]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:03.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:03 smithi145 ceph-mon[26606]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:03.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:03 smithi172 ceph-mon[22747]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:05.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:05 smithi145 ceph-mon[21899]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:05.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:05 smithi145 ceph-mon[26606]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:05.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:05 smithi172 ceph-mon[22747]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:07.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:07 smithi145 ceph-mon[21899]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:07.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:07 smithi145 ceph-mon[26606]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:07.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:07 smithi172 ceph-mon[22747]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:08.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:14:07 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:14:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:14:09.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:09 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:14:09.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:09 smithi145 ceph-mon[21899]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:09.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:09 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:14:09.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:09 smithi145 ceph-mon[26606]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:09.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:09 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:14:09.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:09 smithi172 ceph-mon[22747]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:11.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:11 smithi145 ceph-mon[26606]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:11.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:11 smithi145 ceph-mon[21899]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:11.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:11 smithi172 ceph-mon[22747]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:12.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:14:11 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:14:11] "GET /metrics HTTP/1.1" 200 208000 "" "Prometheus/2.33.4" 2024-04-22T15:14:13.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:13 smithi145 ceph-mon[21899]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:13.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:13 smithi145 ceph-mon[26606]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:13.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:13 smithi172 ceph-mon[22747]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:15.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:15 smithi145 ceph-mon[21899]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:15.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:15 smithi145 ceph-mon[26606]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:15.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:15 smithi172 ceph-mon[22747]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:17.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:17 smithi145 ceph-mon[21899]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:17.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:17 smithi145 ceph-mon[26606]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:17.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:17 smithi172 ceph-mon[22747]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:18.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:14:17 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:14:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:14:19.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:19 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:14:19.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:19 smithi145 ceph-mon[21899]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:19.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:19 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:14:19.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:19 smithi145 ceph-mon[26606]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:19.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:19 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:14:19.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:19 smithi172 ceph-mon[22747]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:21.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:21 smithi145 ceph-mon[21899]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:21.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:21 smithi145 ceph-mon[26606]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:21.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:21 smithi172 ceph-mon[22747]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:22.372 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:14:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:14:21] "GET /metrics HTTP/1.1" 200 207954 "" "Prometheus/2.33.4" 2024-04-22T15:14:23.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:23 smithi145 ceph-mon[21899]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:23.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:23 smithi145 ceph-mon[26606]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:23.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:23 smithi172 ceph-mon[22747]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:25.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:25 smithi145 ceph-mon[26606]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:25.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:25 smithi145 ceph-mon[21899]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:25.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:25 smithi172 ceph-mon[22747]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:27.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:27 smithi145 ceph-mon[26606]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:27.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:27 smithi145 ceph-mon[21899]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:27.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:27 smithi172 ceph-mon[22747]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:28.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:14:27 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:14:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:14:29.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:29 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:14:29.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:29 smithi145 ceph-mon[26606]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:29.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:29 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:14:29.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:29 smithi145 ceph-mon[21899]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:29.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:29 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:14:29.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:29 smithi172 ceph-mon[22747]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:31.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:31 smithi145 ceph-mon[26606]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:31.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:31 smithi145 ceph-mon[21899]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:31.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:31 smithi172 ceph-mon[22747]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:32.372 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:14:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:14:31] "GET /metrics HTTP/1.1" 200 207953 "" "Prometheus/2.33.4" 2024-04-22T15:14:33.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:33 smithi145 ceph-mon[21899]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:33.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:33 smithi145 ceph-mon[26606]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:33.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:33 smithi172 ceph-mon[22747]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:35.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:35 smithi145 ceph-mon[21899]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:35.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:35 smithi145 ceph-mon[26606]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:35.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:35 smithi172 ceph-mon[22747]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:37.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:37 smithi145 ceph-mon[21899]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:37.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:37 smithi145 ceph-mon[26606]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:37.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:37 smithi172 ceph-mon[22747]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:38.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:14:37 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:14:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:14:39.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:39 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:14:39.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:39 smithi145 ceph-mon[26606]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:39.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:39 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:14:39.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:39 smithi145 ceph-mon[21899]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:39.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:39 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:14:39.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:39 smithi172 ceph-mon[22747]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:39.786 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-04-22T15:14:41.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:41 smithi145 ceph-mon[21899]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:41.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:41 smithi145 ceph-mon[26606]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:41.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:41 smithi172 ceph-mon[22747]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:42.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:14:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:14:41] "GET /metrics HTTP/1.1" 200 207953 "" "Prometheus/2.33.4" 2024-04-22T15:14:42.412 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-04-22T15:14:43.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:43 smithi145 ceph-mon[26606]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:43.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:14:43.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:14:43.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:43 smithi145 ceph-mon[21899]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:43.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:14:43.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:14:43.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:43 smithi172 ceph-mon[22747]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:43.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:14:43.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:14:44.934 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph config set global log_to_journald false --force' 2024-04-22T15:14:45.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:45 smithi145 ceph-mon[21899]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:45.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:45 smithi145 ceph-mon[26606]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:45.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:45 smithi172 ceph-mon[22747]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:47.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:47 smithi145 ceph-mon[21899]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:47.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:47 smithi145 ceph-mon[26606]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:47.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:47 smithi172 ceph-mon[22747]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:47.563 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch ps' 2024-04-22T15:14:48.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:14:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:14:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:14:49.122 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:49 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:14:49.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:49 smithi145 ceph-mon[26606]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:49.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:49 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:14:49.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:49 smithi145 ceph-mon[21899]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:49.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:49 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:14:49.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:49 smithi172 ceph-mon[22747]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:49.544 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:14:49.544 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (3m) 2m ago 3m 15.5M - ba2b418f427c 9cbf618f1212 2024-04-22T15:14:49.544 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (3m) 2m ago 3m 44.8M - 8.3.5 dad864ee21e9 b5aa7bda5872 2024-04-22T15:14:49.544 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 running (3m) 2m ago 3m 66.1M - 3.5 e1d6a67b021e cde4babc6c7f 2024-04-22T15:14:49.544 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443 running (8m) 2m ago 8m 410M - 17.2.0 e1d6a67b021e 9701c52446e9 2024-04-22T15:14:49.544 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:9283 running (10m) 2m ago 10m 454M - 17.2.0 e1d6a67b021e 95f50f021d87 2024-04-22T15:14:49.544 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (10m) 2m ago 10m 46.8M 2048M 17.2.0 e1d6a67b021e 28106a7bf16e 2024-04-22T15:14:49.544 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (8m) 2m ago 8m 39.2M 2048M 17.2.0 e1d6a67b021e a2fc3297953a 2024-04-22T15:14:49.544 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (8m) 2m ago 8m 36.9M 2048M 17.2.0 e1d6a67b021e b71d9b9f8509 2024-04-22T15:14:49.544 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (4m) 2m ago 4m 16.3M - 1dbe0e931976 5e2f432a5091 2024-04-22T15:14:49.544 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (4m) 2m ago 4m 16.4M - 1dbe0e931976 759c7e0c3e41 2024-04-22T15:14:49.544 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (8m) 2m ago 8m 49.4M 4016M 17.2.0 e1d6a67b021e ea9cdc639916 2024-04-22T15:14:49.544 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (7m) 2m ago 7m 48.6M 4016M 17.2.0 e1d6a67b021e 3b9b37a8781f 2024-04-22T15:14:49.545 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (7m) 2m ago 7m 45.0M 4016M 17.2.0 e1d6a67b021e d9502dded8f8 2024-04-22T15:14:49.545 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (7m) 2m ago 7m 45.7M 4016M 17.2.0 e1d6a67b021e 8c72f30e54fc 2024-04-22T15:14:49.545 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (6m) 2m ago 6m 48.5M 4272M 17.2.0 e1d6a67b021e 80001f97ed35 2024-04-22T15:14:49.545 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (6m) 2m ago 6m 45.4M 4272M 17.2.0 e1d6a67b021e ec3ab1335b5c 2024-04-22T15:14:49.545 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (5m) 2m ago 5m 45.5M 4272M 17.2.0 e1d6a67b021e 45d475edf1ca 2024-04-22T15:14:49.545 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (5m) 2m ago 5m 44.6M 4272M 17.2.0 e1d6a67b021e cc9e1d9e7916 2024-04-22T15:14:49.545 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (3m) 2m ago 4m 43.4M - 514e6a882f6e 9121b6b1d1de 2024-04-22T15:14:49.545 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (3m) 2m ago 3m 70.9M - 17.2.0 e1d6a67b021e c90a636c947e 2024-04-22T15:14:49.545 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (3m) 2m ago 3m 68.3M - 17.2.0 e1d6a67b021e 20ac46e9bfda 2024-04-22T15:14:50.120 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph versions' 2024-04-22T15:14:50.142 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:50 smithi145 ceph-mon[26606]: from='client.24880 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:14:50.143 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:50 smithi145 ceph-mon[21899]: from='client.24880 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:14:50.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:50 smithi172 ceph-mon[22747]: from='client.24880 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:14:51.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:51 smithi145 ceph-mon[21899]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:51.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:51 smithi145 ceph-mon[26606]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:51.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:51 smithi172 ceph-mon[22747]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:52.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:14:51 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:14:51] "GET /metrics HTTP/1.1" 200 207940 "" "Prometheus/2.33.4" 2024-04-22T15:14:52.240 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:14:52.240 INFO:teuthology.orchestra.run.smithi145.stdout: "mon": { 2024-04-22T15:14:52.240 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-22T15:14:52.240 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:14:52.240 INFO:teuthology.orchestra.run.smithi145.stdout: "mgr": { 2024-04-22T15:14:52.240 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:14:52.240 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:14:52.240 INFO:teuthology.orchestra.run.smithi145.stdout: "osd": { 2024-04-22T15:14:52.240 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T15:14:52.240 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:14:52.241 INFO:teuthology.orchestra.run.smithi145.stdout: "mds": {}, 2024-04-22T15:14:52.241 INFO:teuthology.orchestra.run.smithi145.stdout: "rgw": { 2024-04-22T15:14:52.241 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:14:52.241 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:14:52.241 INFO:teuthology.orchestra.run.smithi145.stdout: "overall": { 2024-04-22T15:14:52.241 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 15 2024-04-22T15:14:52.241 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:14:52.241 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:14:52.592 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:52 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/4013786354' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:14:52.592 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:52 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/4013786354' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:14:52.621 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:52 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/4013786354' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:14:52.917 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph -s' 2024-04-22T15:14:53.587 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:53 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:14:53.587 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:53 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:14:53.587 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:53 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:14:53.588 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:53 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:14:53.588 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:53 smithi145 ceph-mon[26606]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:53.588 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:53 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:14:53.588 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:53 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:14:53.588 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:53 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:14:53.588 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:53 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:14:53.588 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:53 smithi145 ceph-mon[21899]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:53.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:53 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:14:53.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:53 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:14:53.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:53 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:14:53.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:53 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:14:53.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:53 smithi172 ceph-mon[22747]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:55.057 INFO:teuthology.orchestra.run.smithi145.stdout: cluster: 2024-04-22T15:14:55.057 INFO:teuthology.orchestra.run.smithi145.stdout: id: 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:14:55.057 INFO:teuthology.orchestra.run.smithi145.stdout: health: HEALTH_OK 2024-04-22T15:14:55.057 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-04-22T15:14:55.057 INFO:teuthology.orchestra.run.smithi145.stdout: services: 2024-04-22T15:14:55.057 INFO:teuthology.orchestra.run.smithi145.stdout: mon: 3 daemons, quorum a,c,b (age 8m) 2024-04-22T15:14:55.057 INFO:teuthology.orchestra.run.smithi145.stdout: mgr: y(active, since 5m), standbys: x 2024-04-22T15:14:55.057 INFO:teuthology.orchestra.run.smithi145.stdout: osd: 8 osds: 8 up (since 5m), 8 in (since 5m) 2024-04-22T15:14:55.057 INFO:teuthology.orchestra.run.smithi145.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-04-22T15:14:55.057 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-04-22T15:14:55.057 INFO:teuthology.orchestra.run.smithi145.stdout: data: 2024-04-22T15:14:55.057 INFO:teuthology.orchestra.run.smithi145.stdout: pools: 6 pools, 161 pgs 2024-04-22T15:14:55.058 INFO:teuthology.orchestra.run.smithi145.stdout: objects: 209 objects, 585 KiB 2024-04-22T15:14:55.058 INFO:teuthology.orchestra.run.smithi145.stdout: usage: 64 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:14:55.058 INFO:teuthology.orchestra.run.smithi145.stdout: pgs: 161 active+clean 2024-04-22T15:14:55.058 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-04-22T15:14:55.058 INFO:teuthology.orchestra.run.smithi145.stdout: io: 2024-04-22T15:14:55.058 INFO:teuthology.orchestra.run.smithi145.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-04-22T15:14:55.058 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-04-22T15:14:55.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:55 smithi145 ceph-mon[21899]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:55.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:55 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2363920495' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T15:14:55.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:55 smithi145 ceph-mon[26606]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:55.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:55 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2363920495' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T15:14:55.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:55 smithi172 ceph-mon[22747]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:55.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:55 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2363920495' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T15:14:55.740 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch ls' 2024-04-22T15:14:57.260 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:57 smithi145 ceph-mon[26606]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:57.261 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:57 smithi145 ceph-mon[21899]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:57.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:57 smithi172 ceph-mon[22747]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:14:57.651 INFO:teuthology.orchestra.run.smithi145.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-04-22T15:14:57.651 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager ?:9093,9094 1/1 3m ago 5m smithi145=a;count:1 2024-04-22T15:14:57.651 INFO:teuthology.orchestra.run.smithi145.stdout:grafana ?:3000 1/1 3m ago 5m smithi172=a;count:1 2024-04-22T15:14:57.651 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo 1/1 3m ago 3m count:1 2024-04-22T15:14:57.651 INFO:teuthology.orchestra.run.smithi145.stdout:mgr 2/2 3m ago 8m smithi145=y;smithi172=x;count:2 2024-04-22T15:14:57.651 INFO:teuthology.orchestra.run.smithi145.stdout:mon 3/3 3m ago 9m smithi145:172.21.15.145=a;smithi145:[v2:172.21.15.145:3301,v1:172.21.15.145:6790]=c;smithi172:172.21.15.172=b;count:3 2024-04-22T15:14:57.651 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter ?:9100 2/2 3m ago 5m smithi145=a;smithi172=b;count:2 2024-04-22T15:14:57.651 INFO:teuthology.orchestra.run.smithi145.stdout:osd 8 3m ago - 2024-04-22T15:14:57.652 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus ?:9095 1/1 3m ago 5m smithi172=a;count:1 2024-04-22T15:14:57.652 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo ?:8000 2/2 3m ago 3m count:2 2024-04-22T15:14:58.294 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:58 smithi145 ceph-mon[26606]: from='client.24898 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:14:58.295 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:58 smithi145 ceph-mon[21899]: from='client.24898 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:14:58.317 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-22T15:14:58.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:58 smithi172 ceph-mon[22747]: from='client.24898 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:14:58.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:14:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:14:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:14:59.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:59 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:14:59.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:14:59 smithi145 ceph-mon[21899]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:59.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:59 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:14:59.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:14:59 smithi145 ceph-mon[26606]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:14:59.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:59 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:14:59.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:14:59 smithi172 ceph-mon[22747]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:00.684 INFO:teuthology.orchestra.run.smithi145.stdout:Scheduled to redeploy mgr.x on host 'smithi172' 2024-04-22T15:15:00.685 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:00 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2954298691' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-22T15:15:00.685 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:00 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2954298691' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-22T15:15:00.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:00 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2954298691' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-22T15:15:01.340 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch ps --refresh' 2024-04-22T15:15:01.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:01 smithi172 ceph-mon[22747]: from='client.14994 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:15:01.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:01 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:15:01.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:01 smithi172 ceph-mon[22747]: Schedule redeploy daemon mgr.x 2024-04-22T15:15:01.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:01 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:15:01.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:01 smithi172 ceph-mon[22747]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:01.997 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:15:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:15:01] "GET /metrics HTTP/1.1" 200 207944 "" "Prometheus/2.33.4" 2024-04-22T15:15:01.997 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:01 smithi145 ceph-mon[26606]: from='client.14994 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:15:01.997 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:01 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:15:01.997 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:01 smithi145 ceph-mon[26606]: Schedule redeploy daemon mgr.x 2024-04-22T15:15:01.997 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:01 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:15:01.997 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:01 smithi145 ceph-mon[26606]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:01.998 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:01 smithi145 ceph-mon[21899]: from='client.14994 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:15:01.998 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:01 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:15:01.998 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:01 smithi145 ceph-mon[21899]: Schedule redeploy daemon mgr.x 2024-04-22T15:15:01.998 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:01 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:15:01.998 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:01 smithi145 ceph-mon[21899]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:03.175 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:03 smithi145 ceph-mon[26606]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:03.175 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:03 smithi145 ceph-mon[21899]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:03.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:03 smithi172 ceph-mon[22747]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:03.629 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:15:03.629 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (3m) 3m ago 4m 15.5M - ba2b418f427c 9cbf618f1212 2024-04-22T15:15:03.629 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (3m) 3m ago 3m 44.8M - 8.3.5 dad864ee21e9 b5aa7bda5872 2024-04-22T15:15:03.629 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 running (3m) 3m ago 3m 66.1M - 3.5 e1d6a67b021e cde4babc6c7f 2024-04-22T15:15:03.630 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443 running (8m) 3m ago 8m 410M - 17.2.0 e1d6a67b021e 9701c52446e9 2024-04-22T15:15:03.630 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:9283 running (10m) 3m ago 10m 454M - 17.2.0 e1d6a67b021e 95f50f021d87 2024-04-22T15:15:03.630 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (10m) 3m ago 10m 46.8M 2048M 17.2.0 e1d6a67b021e 28106a7bf16e 2024-04-22T15:15:03.630 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (9m) 3m ago 9m 39.2M 2048M 17.2.0 e1d6a67b021e a2fc3297953a 2024-04-22T15:15:03.630 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (9m) 3m ago 9m 36.9M 2048M 17.2.0 e1d6a67b021e b71d9b9f8509 2024-04-22T15:15:03.630 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (4m) 3m ago 4m 16.3M - 1dbe0e931976 5e2f432a5091 2024-04-22T15:15:03.630 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (4m) 3m ago 4m 16.4M - 1dbe0e931976 759c7e0c3e41 2024-04-22T15:15:03.630 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (8m) 3m ago 8m 49.4M 4016M 17.2.0 e1d6a67b021e ea9cdc639916 2024-04-22T15:15:03.630 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (8m) 3m ago 8m 48.6M 4016M 17.2.0 e1d6a67b021e 3b9b37a8781f 2024-04-22T15:15:03.630 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (7m) 3m ago 7m 45.0M 4016M 17.2.0 e1d6a67b021e d9502dded8f8 2024-04-22T15:15:03.630 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (7m) 3m ago 7m 45.7M 4016M 17.2.0 e1d6a67b021e 8c72f30e54fc 2024-04-22T15:15:03.630 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (6m) 3m ago 6m 48.5M 4272M 17.2.0 e1d6a67b021e 80001f97ed35 2024-04-22T15:15:03.630 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (6m) 3m ago 6m 45.4M 4272M 17.2.0 e1d6a67b021e ec3ab1335b5c 2024-04-22T15:15:03.630 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (6m) 3m ago 6m 45.5M 4272M 17.2.0 e1d6a67b021e 45d475edf1ca 2024-04-22T15:15:03.631 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (5m) 3m ago 5m 44.6M 4272M 17.2.0 e1d6a67b021e cc9e1d9e7916 2024-04-22T15:15:03.631 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (3m) 3m ago 5m 43.4M - 514e6a882f6e 9121b6b1d1de 2024-04-22T15:15:03.631 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (3m) 3m ago 3m 70.9M - 17.2.0 e1d6a67b021e c90a636c947e 2024-04-22T15:15:03.631 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (3m) 3m ago 3m 68.3M - 17.2.0 e1d6a67b021e 20ac46e9bfda 2024-04-22T15:15:04.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:04 smithi145 ceph-mon[26606]: from='client.15000 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:15:04.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:04 smithi145 ceph-mon[21899]: from='client.15000 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:15:04.467 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'sleep 180' 2024-04-22T15:15:04.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:04 smithi172 ceph-mon[22747]: from='client.15000 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:15:05.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:05 smithi145 ceph-mon[26606]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:05.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:05 smithi145 ceph-mon[21899]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:05.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:05 smithi172 ceph-mon[22747]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:07.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:07 smithi145 ceph-mon[26606]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:07.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:07 smithi145 ceph-mon[21899]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:07.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:07 smithi172 ceph-mon[22747]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:08.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:15:07 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:15:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:15:09.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:09 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:15:09.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:09 smithi145 ceph-mon[21899]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:09.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:09 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:15:09.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:09 smithi145 ceph-mon[26606]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:09.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:09 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:15:09.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:09 smithi172 ceph-mon[22747]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:11.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:11 smithi145 ceph-mon[21899]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:11.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:11 smithi145 ceph-mon[26606]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:11.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:11 smithi172 ceph-mon[22747]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:12.372 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:15:11 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:15:11] "GET /metrics HTTP/1.1" 200 207944 "" "Prometheus/2.33.4" 2024-04-22T15:15:13.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:13 smithi145 ceph-mon[26606]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:13.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:13 smithi145 ceph-mon[21899]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:13.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:13 smithi172 ceph-mon[22747]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:15.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:15 smithi145 ceph-mon[21899]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:15.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:15 smithi145 ceph-mon[26606]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:15.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:15 smithi172 ceph-mon[22747]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:17.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:17 smithi145 ceph-mon[21899]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:17.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:17 smithi145 ceph-mon[26606]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:17.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:17 smithi172 ceph-mon[22747]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:18.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:15:17 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:15:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:15:19.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:19 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:15:19.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:19 smithi145 ceph-mon[21899]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:19.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:19 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:15:19.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:19 smithi145 ceph-mon[26606]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:19.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:19 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:15:19.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:19 smithi172 ceph-mon[22747]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:21.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:21 smithi145 ceph-mon[26606]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:21.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:21 smithi145 ceph-mon[21899]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:21.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:21 smithi172 ceph-mon[22747]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:22.372 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:15:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:15:21] "GET /metrics HTTP/1.1" 200 207950 "" "Prometheus/2.33.4" 2024-04-22T15:15:23.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:23 smithi145 ceph-mon[26606]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:23.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:23 smithi145 ceph-mon[21899]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:23.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:23 smithi172 ceph-mon[22747]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:25.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:25 smithi145 ceph-mon[26606]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:25.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:25 smithi145 ceph-mon[21899]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:25.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:25 smithi172 ceph-mon[22747]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:27.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:27 smithi145 ceph-mon[26606]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:27.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:27 smithi145 ceph-mon[21899]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:27.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:27 smithi172 ceph-mon[22747]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:28.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:15:27 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:15:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:15:29.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:29 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:15:29.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:29 smithi145 ceph-mon[26606]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:29.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:29 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:15:29.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:29 smithi145 ceph-mon[21899]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:29.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:29 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:15:29.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:29 smithi172 ceph-mon[22747]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:31.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:31 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:15:31.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:31 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:15:31.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:31 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:15:31.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:31 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:15:31.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:31 smithi145 ceph-mon[21899]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:31.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:31 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:15:31.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:31 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:15:31.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:31 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:15:31.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:31 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:15:31.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:31 smithi145 ceph-mon[26606]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:31.662 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:31 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:15:31.662 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:31 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:15:31.663 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:31 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:15:31.663 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:31 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:15:31.663 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:31 smithi172 ceph-mon[22747]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:32.282 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:15:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:15:31] "GET /metrics HTTP/1.1" 200 207951 "" "Prometheus/2.33.4" 2024-04-22T15:15:33.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:33 smithi172 ceph-mon[22747]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:33.595 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:33 smithi145 ceph-mon[21899]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:33.595 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:33 smithi145 ceph-mon[26606]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:35.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:34 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:15:35.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:34 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:15:35.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:34 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:15:35.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:34 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:15:35.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:34 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:15:35.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:34 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:15:35.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:34 smithi145 ceph-mon[21899]: Deploying daemon mgr.x on smithi172 2024-04-22T15:15:35.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:34 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:15:35.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:34 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:15:35.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:34 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:15:35.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:34 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:15:35.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:34 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:15:35.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:34 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:15:35.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:34 smithi145 ceph-mon[26606]: Deploying daemon mgr.x on smithi172 2024-04-22T15:15:35.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:34 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:15:35.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:34 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:15:35.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:34 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:15:35.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:34 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:15:35.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:34 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:15:35.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:34 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:15:35.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:34 smithi172 ceph-mon[22747]: Deploying daemon mgr.x on smithi172 2024-04-22T15:15:36.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:35 smithi145 ceph-mon[21899]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:36.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:35 smithi145 ceph-mon[26606]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:36.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:35 smithi172 ceph-mon[22747]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:37.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:37 smithi145 ceph-mon[21899]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:37.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:37 smithi145 ceph-mon[26606]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:37.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:37 smithi172 ceph-mon[22747]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:38.467 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:15:37 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:15:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:15:39.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:39 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:15:39.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:39 smithi145 ceph-mon[21899]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:39.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:39 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:15:39.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:39 smithi145 ceph-mon[26606]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:39.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:39 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:15:39.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:39 smithi172 ceph-mon[22747]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:41.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:41 smithi172 ceph-mon[22747]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:41.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:41 smithi145 ceph-mon[21899]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:41.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:41 smithi145 ceph-mon[26606]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:42.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:15:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:15:41] "GET /metrics HTTP/1.1" 200 207951 "" "Prometheus/2.33.4" 2024-04-22T15:15:43.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:43 smithi145 ceph-mon[21899]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:43.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:15:43.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:15:43.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:43 smithi145 ceph-mon[26606]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:43.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:15:43.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:15:43.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:43 smithi172 ceph-mon[22747]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:43.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:15:43.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:15:43.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:15:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:15:43.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-04-22T15:15:43.873 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:15:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:15:43.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs" 2024-04-22T15:15:45.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:45 smithi145 ceph-mon[21899]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:45.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:45 smithi145 ceph-mon[26606]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:45.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:45 smithi172 ceph-mon[22747]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:47.423 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:47 smithi145 ceph-mon[26606]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:47.423 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:47 smithi145 ceph-mon[21899]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:47.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:47 smithi172 ceph-mon[22747]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:48.467 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:15:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:15:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:15:49.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:49 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:15:49.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:49 smithi172 ceph-mon[22747]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:49.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:49 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:15:49.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:49 smithi145 ceph-mon[21899]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:49.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:49 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:15:49.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:49 smithi145 ceph-mon[26606]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:51.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:51 smithi172 ceph-mon[22747]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:51.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:51 smithi145 ceph-mon[21899]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:51.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:51 smithi145 ceph-mon[26606]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:52.372 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:15:51 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:15:51] "GET /metrics HTTP/1.1" 200 207957 "" "Prometheus/2.33.4" 2024-04-22T15:15:53.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:53 smithi172 ceph-mon[22747]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:53.509 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:53 smithi145 ceph-mon[21899]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:53.509 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:53 smithi145 ceph-mon[26606]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:53.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:15:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=error ts=2024-04-22T15:15:53.506Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs" 2024-04-22T15:15:53.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:15:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:15:53.508Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs" 2024-04-22T15:15:53.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:15:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:15:53.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-04-22T15:15:55.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:55 smithi145 ceph-mon[21899]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:55.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:55 smithi145 ceph-mon[26606]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:55.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:55 smithi172 ceph-mon[22747]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:57.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:57 smithi172 ceph-mon[22747]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:57.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:57 smithi145 ceph-mon[21899]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:57.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:57 smithi145 ceph-mon[26606]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:15:58.467 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:15:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[23931]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:15:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:15:59.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:59 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:15:59.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:15:59 smithi172 ceph-mon[22747]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:59.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:59 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:15:59.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:15:59 smithi145 ceph-mon[21899]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:15:59.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:59 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:15:59.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:15:59 smithi145 ceph-mon[26606]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:01.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:01 smithi172 ceph-mon[22747]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:01.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:01 smithi145 ceph-mon[21899]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:01.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:01 smithi145 ceph-mon[26606]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:02.372 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:16:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:16:01] "GET /metrics HTTP/1.1" 200 207973 "" "Prometheus/2.33.4" 2024-04-22T15:16:03.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:03 smithi172 ceph-mon[22747]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:03.509 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:03 smithi145 ceph-mon[26606]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:03.509 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:03 smithi145 ceph-mon[21899]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:03.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:16:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=error ts=2024-04-22T15:16:03.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs" 2024-04-22T15:16:03.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:16:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:16:03.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs" 2024-04-22T15:16:03.873 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:16:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:16:03.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-04-22T15:16:04.191 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:03 smithi172 systemd[1]: Stopping Ceph mgr.x for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:16:04.191 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:03 smithi172 bash[42813]: Error: no container with name or ID "ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr.x" found: no such container 2024-04-22T15:16:04.191 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:03 smithi172 podman[42826]: 2024-04-22 15:16:03.91577934 +0000 UTC m=+0.122090929 container died 9701c52446e9331b9aee526b80b064e5d71e43d671024fdcbb6e3f0362166caa (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x, GIT_CLEAN=True, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, version=8, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., name=centos-stream, distribution-scope=public, io.openshift.expose-services=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-04-22T15:16:04.372 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:16:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:16:03.993Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=2 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": dial tcp 172.21.15.172:8443: connect: connection refused" 2024-04-22T15:16:04.467 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:04 smithi172 podman[42826]: 2024-04-22 15:16:04.278212181 +0000 UTC m=+0.484523772 container cleanup 9701c52446e9331b9aee526b80b064e5d71e43d671024fdcbb6e3f0362166caa (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, com.redhat.component=centos-stream-container, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., distribution-scope=public, io.buildah.version=1.19.8, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, version=8, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-22T15:16:04.467 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:04 smithi172 bash[42826]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x 2024-04-22T15:16:04.467 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:04 smithi172 bash[42856]: Error: no container with name or ID "ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr.x" found: no such container 2024-04-22T15:16:04.964 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:04 smithi172 podman[42840]: 2024-04-22 15:16:04.546312692 +0000 UTC m=+0.623431100 container remove 9701c52446e9331b9aee526b80b064e5d71e43d671024fdcbb6e3f0362166caa (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, release=754, vcs-type=git, GIT_CLEAN=True, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, ceph=True) 2024-04-22T15:16:04.964 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:04 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-22T15:16:04.964 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:04 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-04-22T15:16:04.964 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:04 smithi172 systemd[1]: Stopped Ceph mgr.x for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:16:04.964 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:04 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mgr.x.service: Consumed 17.262s CPU time. 2024-04-22T15:16:04.964 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:04 smithi172 systemd[1]: Starting Ceph mgr.x for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:16:05.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:05 smithi172 ceph-mon[22747]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:05.218 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:04 smithi172 podman[42959]: 2024-04-22 15:16:04.864197572 +0000 UTC m=+0.018141218 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:16:05.218 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:05 smithi172 podman[42959]: 2024-04-22 15:16:05.001551123 +0000 UTC m=+0.155494795 container create 1920e9785cbb92f36c806b31a4ea16b6bcfe13b505c10dfd196bc4996397957c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T15:16:05.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:05 smithi145 ceph-mon[21899]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:05.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:05 smithi145 ceph-mon[26606]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:05.475 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:05 smithi172 podman[42959]: 2024-04-22 15:16:05.284545885 +0000 UTC m=+0.438489516 container init 1920e9785cbb92f36c806b31a4ea16b6bcfe13b505c10dfd196bc4996397957c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, ceph=True, org.label-schema.vendor=CentOS) 2024-04-22T15:16:05.475 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:05 smithi172 podman[42959]: 2024-04-22 15:16:05.287333748 +0000 UTC m=+0.441277379 container start 1920e9785cbb92f36c806b31a4ea16b6bcfe13b505c10dfd196bc4996397957c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-04-22T15:16:05.476 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:05 smithi172 ceph-mgr[42978]: -- 172.21.15.172:0/3575880288 <== mon.2 v2:172.21.15.172:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55aeda0225a0 con 0x55aed9ff6c00 2024-04-22T15:16:05.476 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:05 smithi172 bash[42959]: 1920e9785cbb92f36c806b31a4ea16b6bcfe13b505c10dfd196bc4996397957c 2024-04-22T15:16:05.476 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:05 smithi172 systemd[1]: Started Ceph mgr.x for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:16:05.476 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:16:05.446+0000 7fd6e2fc8200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T15:16:05.745 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:16:05.643+0000 7fd6e2fc8200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T15:16:06.104 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:16:05.757+0000 7fd6e2fc8200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T15:16:06.105 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:06 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:16:06.101+0000 7fd6e2fc8200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T15:16:06.413 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:06 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:16:06.187+0000 7fd6e2fc8200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T15:16:06.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:06 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:16:06.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:06 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:16:06.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:06 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:16:06.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:06 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:16:06.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:06 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:16:06.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:06 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:16:06.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:06 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:16:06.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:06 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:16:06.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:06 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:16:06.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:06 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:16:06.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:06 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:16:06.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:06 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:16:06.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:06 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:16:06.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:06 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:16:06.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:06 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:16:06.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:06 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:16:06.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:06 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:16:06.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:06 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:16:07.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:07 smithi172 ceph-mon[22747]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:07.718 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:07 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:16:07.445+0000 7fd6e2fc8200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T15:16:07.718 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:07 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:16:07.643+0000 7fd6e2fc8200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T15:16:07.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:07 smithi145 ceph-mon[21899]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:07.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:07 smithi145 ceph-mon[26606]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:08.080 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:07 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:16:07.820+0000 7fd6e2fc8200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T15:16:08.451 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:08 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:16:08.123+0000 7fd6e2fc8200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T15:16:08.451 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:08 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:16:08.220+0000 7fd6e2fc8200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T15:16:08.717 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:08 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:16:08.448+0000 7fd6e2fc8200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T15:16:08.717 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:08 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:16:08.649+0000 7fd6e2fc8200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T15:16:09.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:09 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:16:09.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:09 smithi172 ceph-mon[22747]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:09.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:09 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:16:09.282+0000 7fd6e2fc8200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T15:16:09.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:09 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:16:09.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:09 smithi145 ceph-mon[21899]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:09.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:09 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:16:09.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:09 smithi145 ceph-mon[26606]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:09.954 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:09 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:16:09.731+0000 7fd6e2fc8200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T15:16:09.954 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:09 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:16:09.835+0000 7fd6e2fc8200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T15:16:10.217 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:10 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:16:10.049+0000 7fd6e2fc8200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-22T15:16:10.920 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:10 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:16:10.920 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:10 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:16:10.921 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:10 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:16:10.578+0000 7fd6e2fc8200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T15:16:10.921 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:10 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:16:10.672+0000 7fd6e2fc8200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T15:16:11.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:10 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:16:11.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:10 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:16:11.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:10 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:16:11.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:10 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:16:11.217 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:10 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:16:10.917+0000 7fd6e2fc8200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T15:16:11.218 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:11 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:16:11.052+0000 7fd6e2fc8200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T15:16:11.218 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:11 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:16:11.151+0000 7fd6e2fc8200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T15:16:11.577 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:11 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:16:11.268+0000 7fd6e2fc8200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T15:16:11.577 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:11 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:16:11.574+0000 7fd6e2fc8200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T15:16:11.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:11 smithi172 ceph-mon[22747]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:12.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:11 smithi145 ceph-mon[21899]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:12.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:11 smithi145 ceph-mon[26606]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:12.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:16:11 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:16:11] "GET /metrics HTTP/1.1" 200 207973 "" "Prometheus/2.33.4" 2024-04-22T15:16:12.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:16:12.184+0000 7fd6e2fc8200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T15:16:12.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:16:12.410+0000 7fd6e2fc8200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T15:16:12.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:16:12] ENGINE Bus STARTING 2024-04-22T15:16:12.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:12 smithi172 ceph-mon[22747]: Standby manager daemon x restarted 2024-04-22T15:16:12.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:12 smithi172 ceph-mon[22747]: Standby manager daemon x started 2024-04-22T15:16:12.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:12 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T15:16:12.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:12 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:16:12.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:12 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T15:16:12.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:12 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:16:12.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: CherryPy Checker: 2024-04-22T15:16:12.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: The Application mounted at '' has an empty config. 2024-04-22T15:16:12.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:16:12.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:16:12] ENGINE Serving on http://:::9283 2024-04-22T15:16:12.969 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:16:12] ENGINE Bus STARTED 2024-04-22T15:16:13.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:12 smithi145 ceph-mon[21899]: Standby manager daemon x restarted 2024-04-22T15:16:13.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:12 smithi145 ceph-mon[21899]: Standby manager daemon x started 2024-04-22T15:16:13.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:12 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T15:16:13.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:12 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:16:13.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:12 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T15:16:13.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:12 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:16:13.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:12 smithi145 ceph-mon[26606]: Standby manager daemon x restarted 2024-04-22T15:16:13.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:12 smithi145 ceph-mon[26606]: Standby manager daemon x started 2024-04-22T15:16:13.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:12 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T15:16:13.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:12 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:16:13.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:12 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T15:16:13.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:12 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:16:13.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:13 smithi145 ceph-mon[26606]: mgrmap e21: y(active, since 6m), standbys: x 2024-04-22T15:16:13.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:13 smithi145 ceph-mon[26606]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:13.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:13 smithi145 ceph-mon[21899]: mgrmap e21: y(active, since 6m), standbys: x 2024-04-22T15:16:13.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:13 smithi145 ceph-mon[21899]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:13.873 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:16:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=error ts=2024-04-22T15:16:13.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": dial tcp 172.21.15.172:8443: connect: connection refused" 2024-04-22T15:16:13.873 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:16:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:16:13.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs" 2024-04-22T15:16:13.873 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:16:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:16:13.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-04-22T15:16:13.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:13 smithi172 ceph-mon[22747]: mgrmap e21: y(active, since 6m), standbys: x 2024-04-22T15:16:13.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:13 smithi172 ceph-mon[22747]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:15.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:15 smithi145 ceph-mon[21899]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:15.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:15 smithi145 ceph-mon[26606]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:15.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:15 smithi172 ceph-mon[22747]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:17.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:17 smithi172 ceph-mon[22747]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:17.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:17 smithi145 ceph-mon[21899]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:17.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:17 smithi145 ceph-mon[26606]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:18.467 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:17 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:16:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:16:19.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:19 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:16:19.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:19 smithi172 ceph-mon[22747]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:19.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:19 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:16:19.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:19 smithi145 ceph-mon[26606]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:19.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:19 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:16:19.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:19 smithi145 ceph-mon[21899]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:21.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:21 smithi172 ceph-mon[22747]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:21.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:21 smithi145 ceph-mon[21899]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:21.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:21 smithi145 ceph-mon[26606]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:22.372 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:16:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:16:21] "GET /metrics HTTP/1.1" 200 207968 "" "Prometheus/2.33.4" 2024-04-22T15:16:23.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:23 smithi172 ceph-mon[22747]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:23.511 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:23 smithi145 ceph-mon[26606]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:23.512 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:23 smithi145 ceph-mon[21899]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:23.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:16:23 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=error ts=2024-04-22T15:16:23.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-04-22T15:16:23.873 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:16:23 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:16:23.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs" 2024-04-22T15:16:23.873 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:16:23 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:16:23.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-04-22T15:16:25.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:25 smithi145 ceph-mon[26606]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:25.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:25 smithi145 ceph-mon[21899]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:25.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:25 smithi172 ceph-mon[22747]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:27.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:27 smithi172 ceph-mon[22747]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:27.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:27 smithi145 ceph-mon[21899]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:27.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:27 smithi145 ceph-mon[26606]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:28.467 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:27 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:16:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:16:29.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:29 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:16:29.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:29 smithi172 ceph-mon[22747]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:29.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:29 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:16:29.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:29 smithi145 ceph-mon[21899]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:29.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:29 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:16:29.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:29 smithi145 ceph-mon[26606]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:31.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:31 smithi172 ceph-mon[22747]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:31.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:31 smithi145 ceph-mon[21899]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:31.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:31 smithi145 ceph-mon[26606]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:32.372 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:16:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:16:31] "GET /metrics HTTP/1.1" 200 207976 "" "Prometheus/2.33.4" 2024-04-22T15:16:33.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:33 smithi172 ceph-mon[22747]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:33.512 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:33 smithi145 ceph-mon[26606]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:33.513 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:33 smithi145 ceph-mon[21899]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:33.873 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:16:33 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=error ts=2024-04-22T15:16:33.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-04-22T15:16:33.873 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:16:33 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:16:33.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs" 2024-04-22T15:16:33.873 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:16:33 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:16:33.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-04-22T15:16:35.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:35 smithi145 ceph-mon[26606]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:35.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:35 smithi145 ceph-mon[21899]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:35.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:35 smithi172 ceph-mon[22747]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:37.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:37 smithi172 ceph-mon[22747]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:37.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:37 smithi145 ceph-mon[21899]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:37 smithi145 ceph-mon[26606]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:38.467 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:37 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:16:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:16:39.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:39 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:16:39.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:39 smithi172 ceph-mon[22747]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:39.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:39 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:16:39.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:39 smithi145 ceph-mon[21899]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:39.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:39 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:16:39.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:39 smithi145 ceph-mon[26606]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:41.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:41 smithi172 ceph-mon[22747]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:41.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:41 smithi145 ceph-mon[26606]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:41.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:41 smithi145 ceph-mon[21899]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:42.372 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:16:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:16:41] "GET /metrics HTTP/1.1" 200 207976 "" "Prometheus/2.33.4" 2024-04-22T15:16:43.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:16:43.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:16:43.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:43 smithi145 ceph-mon[26606]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:43.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:16:43.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:16:43.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:43 smithi145 ceph-mon[21899]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:43.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:16:43.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:16:43.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:43 smithi172 ceph-mon[22747]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:43.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:16:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=error ts=2024-04-22T15:16:43.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-04-22T15:16:43.873 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:16:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:16:43.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs" 2024-04-22T15:16:43.873 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:16:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:16:43.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-04-22T15:16:45.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:45 smithi145 ceph-mon[21899]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:45.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:45 smithi145 ceph-mon[26606]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:45.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:45 smithi172 ceph-mon[22747]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:47.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:47 smithi172 ceph-mon[22747]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:47.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:47 smithi145 ceph-mon[21899]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:47.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:47 smithi145 ceph-mon[26606]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:48.467 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:16:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:16:49.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:49 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:16:49.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:49 smithi172 ceph-mon[22747]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:49.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:49 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:16:49.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:49 smithi145 ceph-mon[21899]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:49.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:49 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:16:49.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:49 smithi145 ceph-mon[26606]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:51.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:51 smithi172 ceph-mon[22747]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:51.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:51 smithi145 ceph-mon[21899]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:51.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:51 smithi145 ceph-mon[26606]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:52.372 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:16:51 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:16:51] "GET /metrics HTTP/1.1" 200 207972 "" "Prometheus/2.33.4" 2024-04-22T15:16:53.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:53 smithi172 ceph-mon[22747]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:53.513 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:53 smithi145 ceph-mon[26606]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:53.513 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:53 smithi145 ceph-mon[21899]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:53.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:16:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=error ts=2024-04-22T15:16:53.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-04-22T15:16:53.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:16:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:16:53.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs" 2024-04-22T15:16:53.873 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:16:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:16:53.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-04-22T15:16:55.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:55 smithi145 ceph-mon[21899]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:55.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:55 smithi145 ceph-mon[26606]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:55.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:55 smithi172 ceph-mon[22747]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:57.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:57 smithi172 ceph-mon[22747]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:57.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:57 smithi145 ceph-mon[26606]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:57.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:57 smithi145 ceph-mon[21899]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:16:58.467 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:16:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:16:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:16:59.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:59 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:16:59.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:16:59 smithi172 ceph-mon[22747]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:59.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:59 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:16:59.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:16:59 smithi145 ceph-mon[26606]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:16:59.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:59 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:16:59.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:16:59 smithi145 ceph-mon[21899]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:01.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:01 smithi172 ceph-mon[22747]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:01.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:01 smithi145 ceph-mon[21899]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:01.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:01 smithi145 ceph-mon[26606]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:02.372 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:17:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:17:01] "GET /metrics HTTP/1.1" 200 207975 "" "Prometheus/2.33.4" 2024-04-22T15:17:03.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:03 smithi172 ceph-mon[22747]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:03.514 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:03 smithi145 ceph-mon[21899]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:03.514 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:03 smithi145 ceph-mon[26606]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:03.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:17:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=error ts=2024-04-22T15:17:03.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-04-22T15:17:03.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:17:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:17:03.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-04-22T15:17:03.873 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:17:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:17:03.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs" 2024-04-22T15:17:05.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:05 smithi145 ceph-mon[26606]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:05.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:05 smithi145 ceph-mon[21899]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:05.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:05 smithi172 ceph-mon[22747]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:07.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:07 smithi172 ceph-mon[22747]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:07.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:07 smithi145 ceph-mon[21899]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:07.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:07 smithi145 ceph-mon[26606]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:08.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:17:07 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:17:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:17:09.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:09 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:17:09.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:09 smithi172 ceph-mon[22747]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:09.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:09 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:17:09.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:09 smithi145 ceph-mon[21899]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:09.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:09 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:17:09.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:09 smithi145 ceph-mon[26606]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:10.268 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:10 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:17:10.268 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:10 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:10.268 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:10 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:17:10.268 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:10 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:17:10.268 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:10 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:10.268 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:10 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:17:10.274 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:10 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:17:10.274 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:10 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:17:10.274 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:10 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:17:11.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:11 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:17:11.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:11 smithi145 ceph-mon[21899]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:11.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:11 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:17:11.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:11 smithi145 ceph-mon[26606]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:11.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:11 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:17:11.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:11 smithi172 ceph-mon[22747]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:12.372 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:17:11 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:17:11] "GET /metrics HTTP/1.1" 200 207975 "" "Prometheus/2.33.4" 2024-04-22T15:17:13.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:13 smithi172 ceph-mon[22747]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:13.514 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:13 smithi145 ceph-mon[26606]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:13.514 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:13 smithi145 ceph-mon[21899]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:13.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:17:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=error ts=2024-04-22T15:17:13.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs" 2024-04-22T15:17:13.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:17:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:17:13.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs" 2024-04-22T15:17:13.873 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:17:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:17:13.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-04-22T15:17:15.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:15 smithi145 ceph-mon[21899]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:15.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:15 smithi145 ceph-mon[26606]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:15.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:15 smithi172 ceph-mon[22747]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:17.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:17 smithi172 ceph-mon[22747]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:17.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:17 smithi145 ceph-mon[26606]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:17.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:17 smithi145 ceph-mon[21899]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:18.467 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:17:17 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:17:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:17:19.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:19 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:17:19.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:19 smithi172 ceph-mon[22747]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:19.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:19 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:17:19.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:19 smithi145 ceph-mon[21899]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:19.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:19 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:17:19.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:19 smithi145 ceph-mon[26606]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:21.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:21 smithi172 ceph-mon[22747]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:21.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:21 smithi145 ceph-mon[21899]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:21.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:21 smithi145 ceph-mon[26606]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:22.372 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:17:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:17:21] "GET /metrics HTTP/1.1" 200 207971 "" "Prometheus/2.33.4" 2024-04-22T15:17:23.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:23 smithi172 ceph-mon[22747]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:23.514 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:23 smithi145 ceph-mon[21899]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:23.514 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:23 smithi145 ceph-mon[26606]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:23.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:17:23 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=error ts=2024-04-22T15:17:23.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-04-22T15:17:23.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:17:23 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:17:23.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs" 2024-04-22T15:17:23.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:17:23 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:17:23.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-04-22T15:17:25.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:25 smithi145 ceph-mon[21899]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:25.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:25 smithi145 ceph-mon[26606]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:25.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:25 smithi172 ceph-mon[22747]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:27.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:27 smithi172 ceph-mon[22747]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:27.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:27 smithi145 ceph-mon[21899]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:27.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:27 smithi145 ceph-mon[26606]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:28.467 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:17:27 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:17:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:17:29.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:29 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:17:29.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:29 smithi172 ceph-mon[22747]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:29.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:29 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:17:29.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:29 smithi145 ceph-mon[21899]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:29.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:29 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:17:29.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:29 smithi145 ceph-mon[26606]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:31.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:31 smithi172 ceph-mon[22747]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:31.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:31 smithi145 ceph-mon[21899]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:31.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:31 smithi145 ceph-mon[26606]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:32.372 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:17:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:17:31] "GET /metrics HTTP/1.1" 200 207967 "" "Prometheus/2.33.4" 2024-04-22T15:17:33.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:33 smithi172 ceph-mon[22747]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:33.515 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:33 smithi145 ceph-mon[26606]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:33.515 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:33 smithi145 ceph-mon[21899]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:33.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:17:33 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=error ts=2024-04-22T15:17:33.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-04-22T15:17:33.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:17:33 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:17:33.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs" 2024-04-22T15:17:33.873 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:17:33 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:17:33.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-04-22T15:17:35.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:35 smithi145 ceph-mon[21899]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:35.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:35 smithi145 ceph-mon[26606]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:35.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:35 smithi172 ceph-mon[22747]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:37.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:37 smithi172 ceph-mon[22747]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:37.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:37 smithi145 ceph-mon[21899]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:37 smithi145 ceph-mon[26606]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:38.467 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:17:37 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:17:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:17:39.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:39 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:17:39.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:39 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:17:39.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:39 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:17:40.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:40 smithi172 ceph-mon[22747]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:40.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:40 smithi145 ceph-mon[21899]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:40.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:40 smithi145 ceph-mon[26606]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:42.155 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:17:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:17:41] "GET /metrics HTTP/1.1" 200 207967 "" "Prometheus/2.33.4" 2024-04-22T15:17:42.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:42 smithi172 ceph-mon[22747]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:42.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:42 smithi145 ceph-mon[21899]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:42.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:42 smithi145 ceph-mon[26606]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:43.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:17:43.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:43 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:17:43.516 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:17:43.516 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:43 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:17:43.516 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:17:43.517 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:43 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:17:43.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:17:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=error ts=2024-04-22T15:17:43.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs" 2024-04-22T15:17:43.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:17:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:17:43.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs" 2024-04-22T15:17:43.873 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:17:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:17:43.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-04-22T15:17:44.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:44 smithi172 ceph-mon[22747]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:44.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:44 smithi145 ceph-mon[21899]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:44.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:44 smithi145 ceph-mon[26606]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:46.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:46 smithi172 ceph-mon[22747]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:46.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:46 smithi145 ceph-mon[21899]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:46.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:46 smithi145 ceph-mon[26606]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:48.425 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:48 smithi145 ceph-mon[26606]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:48.425 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:48 smithi145 ceph-mon[21899]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:48.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:48 smithi172 ceph-mon[22747]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:48.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:17:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:17:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:17:49.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:49 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:17:49.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:49 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:17:49.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:49 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:17:50.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:50 smithi172 ceph-mon[22747]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:50.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:50 smithi145 ceph-mon[21899]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:50.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:50 smithi145 ceph-mon[26606]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:52.178 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:17:51 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:17:51] "GET /metrics HTTP/1.1" 200 207967 "" "Prometheus/2.33.4" 2024-04-22T15:17:52.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:52 smithi172 ceph-mon[22747]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:52.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:52 smithi145 ceph-mon[21899]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:52.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:52 smithi145 ceph-mon[26606]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:53.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:17:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=error ts=2024-04-22T15:17:53.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs" 2024-04-22T15:17:53.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:17:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:17:53.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs" 2024-04-22T15:17:53.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:17:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:17:53.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-04-22T15:17:54.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:54 smithi172 ceph-mon[22747]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:54.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:54 smithi145 ceph-mon[21899]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:54.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:54 smithi145 ceph-mon[26606]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:56.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:56 smithi172 ceph-mon[22747]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:56.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:56 smithi145 ceph-mon[21899]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:56.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:56 smithi145 ceph-mon[26606]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:17:58.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:58 smithi172 ceph-mon[22747]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:58.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:17:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:17:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:17:58.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:58 smithi145 ceph-mon[21899]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:58.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:58 smithi145 ceph-mon[26606]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:17:59.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:17:59 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:17:59.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:17:59 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:17:59.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:17:59 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:18:00.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:00 smithi172 ceph-mon[22747]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:18:00.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:00 smithi145 ceph-mon[21899]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:18:00.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:00 smithi145 ceph-mon[26606]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:18:02.196 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:18:01] "GET /metrics HTTP/1.1" 200 207964 "" "Prometheus/2.33.4" 2024-04-22T15:18:02.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:02 smithi172 ceph-mon[22747]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:18:02.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:02 smithi145 ceph-mon[26606]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:18:02.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:02 smithi145 ceph-mon[21899]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:18:03.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=error ts=2024-04-22T15:18:03.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs" 2024-04-22T15:18:03.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:18:03.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs" 2024-04-22T15:18:03.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:18:03.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-04-22T15:18:04.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:04 smithi172 ceph-mon[22747]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:18:04.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:04 smithi145 ceph-mon[26606]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:18:04.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:04 smithi145 ceph-mon[21899]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:18:06.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:06 smithi172 ceph-mon[22747]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:18:06.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:06 smithi145 ceph-mon[21899]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:18:06.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:06 smithi145 ceph-mon[26606]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:18:08.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:08 smithi172 ceph-mon[22747]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:18:08.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:07 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:18:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:18:08.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:08 smithi145 ceph-mon[26606]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:18:08.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:08 smithi145 ceph-mon[21899]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:18:09.009 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch ps' 2024-04-22T15:18:09.304 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:09 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:18:09.304 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:09 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:18:09.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:09 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:18:10.314 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:10 smithi145 ceph-mon[26606]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:18:10.314 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:10 smithi145 ceph-mon[21899]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:18:10.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:10 smithi172 ceph-mon[22747]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:18:11.015 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:18:11.015 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (6m) 2m ago 7m 24.0M - ba2b418f427c 9cbf618f1212 2024-04-22T15:18:11.016 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (6m) 2m ago 6m 53.6M - 8.3.5 dad864ee21e9 b5aa7bda5872 2024-04-22T15:18:11.016 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 running (6m) 2m ago 6m 51.1M - 3.5 e1d6a67b021e cde4babc6c7f 2024-04-22T15:18:11.016 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443,9283 running (2m) 2m ago 11m 62.2M - 19.0.0-3223-g5883d494 3d983c26e278 1920e9785cbb 2024-04-22T15:18:11.016 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:9283 running (13m) 2m ago 13m 459M - 17.2.0 e1d6a67b021e 95f50f021d87 2024-04-22T15:18:11.016 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (13m) 2m ago 13m 50.0M 2048M 17.2.0 e1d6a67b021e 28106a7bf16e 2024-04-22T15:18:11.016 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (12m) 2m ago 12m 38.3M 2048M 17.2.0 e1d6a67b021e a2fc3297953a 2024-04-22T15:18:11.016 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (12m) 2m ago 12m 36.8M 2048M 17.2.0 e1d6a67b021e b71d9b9f8509 2024-04-22T15:18:11.016 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (8m) 2m ago 8m 16.9M - 1dbe0e931976 5e2f432a5091 2024-04-22T15:18:11.016 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (7m) 2m ago 7m 21.2M - 1dbe0e931976 759c7e0c3e41 2024-04-22T15:18:11.016 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (11m) 2m ago 11m 51.7M 4016M 17.2.0 e1d6a67b021e ea9cdc639916 2024-04-22T15:18:11.016 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (11m) 2m ago 11m 50.2M 4016M 17.2.0 e1d6a67b021e 3b9b37a8781f 2024-04-22T15:18:11.016 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (10m) 2m ago 10m 47.2M 4016M 17.2.0 e1d6a67b021e d9502dded8f8 2024-04-22T15:18:11.016 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (10m) 2m ago 10m 48.7M 4016M 17.2.0 e1d6a67b021e 8c72f30e54fc 2024-04-22T15:18:11.016 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (10m) 2m ago 10m 51.6M 4272M 17.2.0 e1d6a67b021e 80001f97ed35 2024-04-22T15:18:11.016 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (9m) 2m ago 9m 49.1M 4272M 17.2.0 e1d6a67b021e ec3ab1335b5c 2024-04-22T15:18:11.017 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (9m) 2m ago 9m 48.4M 4272M 17.2.0 e1d6a67b021e 45d475edf1ca 2024-04-22T15:18:11.017 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (8m) 2m ago 8m 48.0M 4272M 17.2.0 e1d6a67b021e cc9e1d9e7916 2024-04-22T15:18:11.017 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (6m) 2m ago 8m 67.0M - 514e6a882f6e 9121b6b1d1de 2024-04-22T15:18:11.017 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (6m) 2m ago 6m 71.9M - 17.2.0 e1d6a67b021e c90a636c947e 2024-04-22T15:18:11.017 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (6m) 2m ago 6m 69.5M - 17.2.0 e1d6a67b021e 20ac46e9bfda 2024-04-22T15:18:11.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:11 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:18:11.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:11 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:18:11.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:11 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:18:11.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:11 smithi172 ceph-mon[22747]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:18:11.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:11 smithi172 ceph-mon[22747]: from='client.24928 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:18:11.585 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:11 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:18:11.585 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:11 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:18:11.585 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:11 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:18:11.585 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:11 smithi145 ceph-mon[26606]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:18:11.585 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:11 smithi145 ceph-mon[26606]: from='client.24928 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:18:11.586 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:11 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:18:11.586 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:11 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:18:11.586 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:11 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:18:11.586 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:11 smithi145 ceph-mon[21899]: from='mgr.14448 172.21.15.145:0/1987462468' entity='mgr.y' 2024-04-22T15:18:11.586 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:11 smithi145 ceph-mon[21899]: from='client.24928 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:18:11.676 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph versions' 2024-04-22T15:18:12.218 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:11 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:18:11] "GET /metrics HTTP/1.1" 200 207964 "" "Prometheus/2.33.4" 2024-04-22T15:18:12.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:12 smithi172 ceph-mon[22747]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:18:12.984 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:12 smithi145 ceph-mon[26606]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:18:12.985 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:12 smithi145 ceph-mon[21899]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:18:13.738 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:18:13.738 INFO:teuthology.orchestra.run.smithi145.stdout: "mon": { 2024-04-22T15:18:13.738 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-22T15:18:13.738 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:18:13.738 INFO:teuthology.orchestra.run.smithi145.stdout: "mgr": { 2024-04-22T15:18:13.738 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-22T15:18:13.738 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 1 2024-04-22T15:18:13.738 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:18:13.738 INFO:teuthology.orchestra.run.smithi145.stdout: "osd": { 2024-04-22T15:18:13.738 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T15:18:13.739 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:18:13.739 INFO:teuthology.orchestra.run.smithi145.stdout: "mds": {}, 2024-04-22T15:18:13.739 INFO:teuthology.orchestra.run.smithi145.stdout: "rgw": { 2024-04-22T15:18:13.739 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:18:13.739 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:18:13.739 INFO:teuthology.orchestra.run.smithi145.stdout: "overall": { 2024-04-22T15:18:13.739 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14, 2024-04-22T15:18:13.739 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 1 2024-04-22T15:18:13.739 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:18:13.739 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:18:13.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=error ts=2024-04-22T15:18:13.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs" 2024-04-22T15:18:13.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:18:13.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-04-22T15:18:13.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:18:13.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs" 2024-04-22T15:18:14.441 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph -s' 2024-04-22T15:18:14.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:14 smithi172 ceph-mon[22747]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:18:14.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:14 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2779288027' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:18:14.988 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:14 smithi145 ceph-mon[26606]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:18:14.988 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:14 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2779288027' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:18:14.988 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:14 smithi145 ceph-mon[21899]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:18:14.988 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:14 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2779288027' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:18:16.441 INFO:teuthology.orchestra.run.smithi145.stdout: cluster: 2024-04-22T15:18:16.442 INFO:teuthology.orchestra.run.smithi145.stdout: id: 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:18:16.442 INFO:teuthology.orchestra.run.smithi145.stdout: health: HEALTH_OK 2024-04-22T15:18:16.442 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-04-22T15:18:16.442 INFO:teuthology.orchestra.run.smithi145.stdout: services: 2024-04-22T15:18:16.442 INFO:teuthology.orchestra.run.smithi145.stdout: mon: 3 daemons, quorum a,c,b (age 12m) 2024-04-22T15:18:16.442 INFO:teuthology.orchestra.run.smithi145.stdout: mgr: y(active, since 8m), standbys: x 2024-04-22T15:18:16.442 INFO:teuthology.orchestra.run.smithi145.stdout: osd: 8 osds: 8 up (since 8m), 8 in (since 9m) 2024-04-22T15:18:16.442 INFO:teuthology.orchestra.run.smithi145.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-04-22T15:18:16.442 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-04-22T15:18:16.442 INFO:teuthology.orchestra.run.smithi145.stdout: data: 2024-04-22T15:18:16.442 INFO:teuthology.orchestra.run.smithi145.stdout: pools: 6 pools, 161 pgs 2024-04-22T15:18:16.442 INFO:teuthology.orchestra.run.smithi145.stdout: objects: 209 objects, 585 KiB 2024-04-22T15:18:16.443 INFO:teuthology.orchestra.run.smithi145.stdout: usage: 64 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:18:16.443 INFO:teuthology.orchestra.run.smithi145.stdout: pgs: 161 active+clean 2024-04-22T15:18:16.443 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-04-22T15:18:16.443 INFO:teuthology.orchestra.run.smithi145.stdout: io: 2024-04-22T15:18:16.443 INFO:teuthology.orchestra.run.smithi145.stdout: client: 1.2 KiB/s rd, 1 op/s rd, 0 op/s wr 2024-04-22T15:18:16.443 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-04-22T15:18:16.780 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:16 smithi145 ceph-mon[26606]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:18:16.780 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:16 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/924380729' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T15:18:16.780 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:16 smithi145 ceph-mon[21899]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:18:16.780 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:16 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/924380729' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T15:18:16.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:16 smithi172 ceph-mon[22747]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:18:16.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:16 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/924380729' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T15:18:17.097 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph health detail' 2024-04-22T15:18:18.467 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:17 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:18:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:18:18.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:18 smithi172 ceph-mon[22747]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:18:19.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:18 smithi145 ceph-mon[21899]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:18:19.122 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:18 smithi145 ceph-mon[26606]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:18:19.131 INFO:teuthology.orchestra.run.smithi145.stdout:HEALTH_OK 2024-04-22T15:18:19.755 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:19 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:18:19.755 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:19 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/842661424' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:18:19.755 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:19 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:18:19.755 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:19 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/842661424' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:18:19.779 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 2'"'"'' 2024-04-22T15:18:19.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:19 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:18:19.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:19 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/842661424' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:18:20.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:20 smithi172 ceph-mon[22747]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:18:20.970 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:20 smithi145 ceph-mon[26606]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:18:20.970 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:20 smithi145 ceph-mon[21899]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:18:21.805 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:18:22.122 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:18:21] "GET /metrics HTTP/1.1" 200 207970 "" "Prometheus/2.33.4" 2024-04-22T15:18:22.434 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph mgr fail' 2024-04-22T15:18:22.964 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:22 smithi145 ceph-mon[21899]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:18:22.964 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:22 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3752753657' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:18:22.964 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:22 smithi145 ceph-mon[26606]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:18:22.964 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:22 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3752753657' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:18:22.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:22 smithi172 ceph-mon[22747]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:18:22.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:22 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/3752753657' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:18:23.701 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:23 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=error ts=2024-04-22T15:18:23.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs" 2024-04-22T15:18:23.701 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:23 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:18:23.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs" 2024-04-22T15:18:23.701 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:23 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:18:23.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-04-22T15:18:24.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:24 smithi172 ceph-mon[22747]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:18:24.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:24 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2832316839' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-22T15:18:24.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:24 smithi172 ceph-mon[22747]: osdmap e97: 8 total, 8 up, 8 in 2024-04-22T15:18:25.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:24 smithi145 ceph-mon[21899]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:18:25.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:24 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2832316839' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-22T15:18:25.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:24 smithi145 ceph-mon[21899]: osdmap e97: 8 total, 8 up, 8 in 2024-04-22T15:18:25.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:24 smithi145 ceph-mon[26606]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:18:25.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:24 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2832316839' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-22T15:18:25.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:24 smithi145 ceph-mon[26606]: osdmap e97: 8 total, 8 up, 8 in 2024-04-22T15:18:25.440 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:18:25.158Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=3 err="Post \"https://172.21.15.145:8443//api/prometheus_receiver\": read tcp 172.21.15.145:57788->172.21.15.145:8443: read: connection reset by peer" 2024-04-22T15:18:25.441 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ignoring --setuser ceph since I am not root 2024-04-22T15:18:25.441 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ignoring --setgroup ceph since I am not root 2024-04-22T15:18:25.441 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mgr[22135]: -- 172.21.15.145:0/2911604811 <== mon.0 v2:172.21.15.145:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ead2d9a340 con 0x55ead2d72c00 2024-04-22T15:18:25.441 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:18:25.303+0000 7f5ec2f5d000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T15:18:25.441 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:18:25.437+0000 7f5ec2f5d000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T15:18:25.467 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:18:25] ENGINE Bus STOPPING 2024-04-22T15:18:25.467 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:18:25] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-22T15:18:25.467 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:18:25] ENGINE Bus STOPPED 2024-04-22T15:18:25.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:18:25] ENGINE Bus STARTING 2024-04-22T15:18:25.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:18:25] ENGINE Serving on http://:::9283 2024-04-22T15:18:25.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:18:25] ENGINE Bus STARTED 2024-04-22T15:18:25.706 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[21899]: Standby manager daemon y started 2024-04-22T15:18:25.706 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2832316839' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-22T15:18:25.707 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:18:25.537+0000 7f5ec2f5d000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T15:18:25.861 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'sleep 180' 2024-04-22T15:18:25.964 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[26606]: Standby manager daemon y started 2024-04-22T15:18:25.964 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2832316839' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-22T15:18:25.964 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[26606]: mgrmap e22: x(active, starting, since 0.723092s), standbys: y 2024-04-22T15:18:25.965 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:18:25.965 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:18:25.965 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:18:25.965 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T15:18:25.965 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T15:18:25.965 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:18:25.965 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:18:25.965 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:18:25.965 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:18:25.965 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:18:25.965 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:18:25.965 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:18:25.965 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:18:25.965 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:18:25.966 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:18:25.966 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:18:25.966 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[26606]: Manager daemon x is now available 2024-04-22T15:18:25.966 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:25.966 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[26606]: Queued rgw.foo for migration 2024-04-22T15:18:25.966 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[26606]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-04-22T15:18:25.966 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:25.966 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:18:25.966 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:18:25.966 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:18:25.966 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:18:25.966 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-22T15:18:25.967 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-22T15:18:25.967 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:25.967 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:25.967 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[21899]: mgrmap e22: x(active, starting, since 0.723092s), standbys: y 2024-04-22T15:18:25.967 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:18:25.967 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:18:25.968 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:18:25.968 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T15:18:25.968 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T15:18:25.968 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:18:25.968 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:18:25.968 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:18:25.968 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:18:25.968 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:18:25.968 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:18:25.968 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:18:25.968 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:18:25.968 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:18:25.968 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:18:25.968 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:18:25.969 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[21899]: Manager daemon x is now available 2024-04-22T15:18:25.969 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:25.969 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[21899]: Queued rgw.foo for migration 2024-04-22T15:18:25.969 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[21899]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-04-22T15:18:25.969 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:25.969 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:18:25.969 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:18:25.969 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:18:25.969 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:18:25.969 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-22T15:18:25.969 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-22T15:18:25.969 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:25.969 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:25.970 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:18:25.866+0000 7f5ec2f5d000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T15:18:25.970 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:18:25.960+0000 7f5ec2f5d000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T15:18:25.970 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-mon[22747]: Standby manager daemon y started 2024-04-22T15:18:25.970 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2832316839' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-22T15:18:25.970 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-mon[22747]: mgrmap e22: x(active, starting, since 0.723092s), standbys: y 2024-04-22T15:18:25.970 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:18:25.971 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:18:25.971 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:18:25.971 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T15:18:25.971 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T15:18:25.971 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:18:25.971 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:18:25.971 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:18:25.971 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:18:25.971 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:18:25.971 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:18:25.971 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:18:25.971 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:18:25.971 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:18:25.971 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:18:25.972 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:18:25.972 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-mon[22747]: Manager daemon x is now available 2024-04-22T15:18:25.972 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:25.972 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-mon[22747]: Queued rgw.foo for migration 2024-04-22T15:18:25.972 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-mon[22747]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-04-22T15:18:25.972 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:25.972 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:18:25.972 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:18:25.972 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:18:25.972 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:18:25.972 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-22T15:18:25.972 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-22T15:18:25.972 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:25.972 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:25 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:26.530 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:18:26.224Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=4 err="Post \"https://172.21.15.145:8443//api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-04-22T15:18:27.407 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:18:27.137+0000 7f5ec2f5d000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T15:18:27.408 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:18:27.320+0000 7f5ec2f5d000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T15:18:27.408 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:18:27.404+0000 7f5ec2f5d000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T15:18:27.408 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:27 smithi145 ceph-mon[26606]: Deploying cephadm binary to smithi172 2024-04-22T15:18:27.408 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:27 smithi145 ceph-mon[26606]: Deploying cephadm binary to smithi145 2024-04-22T15:18:27.408 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:27 smithi145 ceph-mon[26606]: [22/Apr/2024:15:18:25] ENGINE Bus STARTING 2024-04-22T15:18:27.408 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:27 smithi145 ceph-mon[26606]: [22/Apr/2024:15:18:26] ENGINE Serving on http://172.21.15.172:8765 2024-04-22T15:18:27.408 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:27 smithi145 ceph-mon[26606]: mgrmap e23: x(active, since 1.7238s), standbys: y 2024-04-22T15:18:27.409 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:27 smithi145 ceph-mon[26606]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:18:27.409 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:27 smithi145 ceph-mon[26606]: [22/Apr/2024:15:18:26] ENGINE Serving on https://172.21.15.172:7150 2024-04-22T15:18:27.409 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:27 smithi145 ceph-mon[26606]: [22/Apr/2024:15:18:26] ENGINE Bus STARTED 2024-04-22T15:18:27.409 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:27 smithi145 ceph-mon[21899]: Deploying cephadm binary to smithi172 2024-04-22T15:18:27.409 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:27 smithi145 ceph-mon[21899]: Deploying cephadm binary to smithi145 2024-04-22T15:18:27.409 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:27 smithi145 ceph-mon[21899]: [22/Apr/2024:15:18:25] ENGINE Bus STARTING 2024-04-22T15:18:27.409 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:27 smithi145 ceph-mon[21899]: [22/Apr/2024:15:18:26] ENGINE Serving on http://172.21.15.172:8765 2024-04-22T15:18:27.410 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:27 smithi145 ceph-mon[21899]: mgrmap e23: x(active, since 1.7238s), standbys: y 2024-04-22T15:18:27.410 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:27 smithi145 ceph-mon[21899]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:18:27.410 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:27 smithi145 ceph-mon[21899]: [22/Apr/2024:15:18:26] ENGINE Serving on https://172.21.15.172:7150 2024-04-22T15:18:27.410 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:27 smithi145 ceph-mon[21899]: [22/Apr/2024:15:18:26] ENGINE Bus STARTED 2024-04-22T15:18:27.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:27 smithi172 ceph-mon[22747]: Deploying cephadm binary to smithi172 2024-04-22T15:18:27.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:27 smithi172 ceph-mon[22747]: Deploying cephadm binary to smithi145 2024-04-22T15:18:27.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:27 smithi172 ceph-mon[22747]: [22/Apr/2024:15:18:25] ENGINE Bus STARTING 2024-04-22T15:18:27.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:27 smithi172 ceph-mon[22747]: [22/Apr/2024:15:18:26] ENGINE Serving on http://172.21.15.172:8765 2024-04-22T15:18:27.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:27 smithi172 ceph-mon[22747]: mgrmap e23: x(active, since 1.7238s), standbys: y 2024-04-22T15:18:27.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:27 smithi172 ceph-mon[22747]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:18:27.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:27 smithi172 ceph-mon[22747]: [22/Apr/2024:15:18:26] ENGINE Serving on https://172.21.15.172:7150 2024-04-22T15:18:27.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:27 smithi172 ceph-mon[22747]: [22/Apr/2024:15:18:26] ENGINE Bus STARTED 2024-04-22T15:18:28.122 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:18:27.725+0000 7f5ec2f5d000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T15:18:28.122 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:18:27.815+0000 7f5ec2f5d000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T15:18:28.122 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:18:27.908+0000 7f5ec2f5d000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T15:18:28.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:28 smithi172 ceph-mon[22747]: mgrmap e24: x(active, since 2s), standbys: y 2024-04-22T15:18:28.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:28 smithi172 ceph-mon[22747]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:18:28.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:27 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:18:27] "GET /metrics HTTP/1.1" 200 35108 "" "Prometheus/2.33.4" 2024-04-22T15:18:28.602 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:28 smithi145 ceph-mon[21899]: mgrmap e24: x(active, since 2s), standbys: y 2024-04-22T15:18:28.602 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:28 smithi145 ceph-mon[21899]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:18:28.603 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:28 smithi145 ceph-mon[26606]: mgrmap e24: x(active, since 2s), standbys: y 2024-04-22T15:18:28.603 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:28 smithi145 ceph-mon[26606]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:18:28.603 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:18:28.156+0000 7f5ec2f5d000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T15:18:28.872 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:18:28.599+0000 7f5ec2f5d000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T15:18:28.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:28 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:18:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: |\n OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked down and back up at {{ $value | humanize }} times once a minute for 5 minutes. This could indicate a network issue (latency, packet drop, disruption) on the clusters \"cluster network\". Check the network environment on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSD's to flap (mark each other out)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"172.21.15.145:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:18:28.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:28 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:18:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: |\n OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by more than 30% from average PG count.\n summary: PG allocations are not balanced across devices\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"172.21.15.145:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:18:29.372 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:18:29.090+0000 7f5ec2f5d000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T15:18:29.372 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:18:29.209+0000 7f5ec2f5d000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T15:18:29.872 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:18:29.510+0000 7f5ec2f5d000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T15:18:29.872 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:18:29.600+0000 7f5ec2f5d000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T15:18:30.370 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:30 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:18:30.370 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:30 smithi145 ceph-mon[26606]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:18:30.370 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:30 smithi145 ceph-mon[26606]: mgrmap e25: x(active, since 4s), standbys: y 2024-04-22T15:18:30.371 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:30 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:18:30.371 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:30 smithi145 ceph-mon[21899]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:18:30.371 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:30 smithi145 ceph-mon[21899]: mgrmap e25: x(active, since 4s), standbys: y 2024-04-22T15:18:30.371 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:18:30.017+0000 7f5ec2f5d000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T15:18:30.371 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:18:30.153+0000 7f5ec2f5d000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T15:18:30.371 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:18:30.248+0000 7f5ec2f5d000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T15:18:30.371 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:18:30.366+0000 7f5ec2f5d000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T15:18:30.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:30 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:18:30.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:30 smithi172 ceph-mon[22747]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:18:30.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:30 smithi172 ceph-mon[22747]: mgrmap e25: x(active, since 4s), standbys: y 2024-04-22T15:18:30.622 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:18:30.462+0000 7f5ec2f5d000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T15:18:31.372 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:18:31.057+0000 7f5ec2f5d000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T15:18:31.872 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:18:31.433+0000 7f5ec2f5d000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T15:18:31.872 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: [22/Apr/2024:15:18:31] ENGINE Bus STARTING 2024-04-22T15:18:31.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: CherryPy Checker: 2024-04-22T15:18:31.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: The Application mounted at '' has an empty config. 2024-04-22T15:18:31.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: 2024-04-22T15:18:31.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: [22/Apr/2024:15:18:31] ENGINE Serving on http://:::9283 2024-04-22T15:18:31.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: [22/Apr/2024:15:18:31] ENGINE Bus STARTED 2024-04-22T15:18:32.146 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:18:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:18:32.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:32 smithi172 ceph-mon[22747]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:18:32.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:32 smithi172 ceph-mon[22747]: Standby manager daemon y restarted 2024-04-22T15:18:32.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:32 smithi172 ceph-mon[22747]: Standby manager daemon y started 2024-04-22T15:18:32.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:32 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.145:0/2165963353' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-22T15:18:32.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:32 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.145:0/2165963353' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:18:32.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:32 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.145:0/2165963353' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-22T15:18:32.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:32 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.145:0/2165963353' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:18:32.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:32 smithi145 ceph-mon[21899]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:18:32.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:32 smithi145 ceph-mon[21899]: Standby manager daemon y restarted 2024-04-22T15:18:32.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:32 smithi145 ceph-mon[21899]: Standby manager daemon y started 2024-04-22T15:18:32.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:32 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.145:0/2165963353' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-22T15:18:32.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:32 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.145:0/2165963353' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:18:32.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:32 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.145:0/2165963353' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-22T15:18:32.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:32 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.145:0/2165963353' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:18:32.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:32 smithi145 ceph-mon[26606]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:18:32.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:32 smithi145 ceph-mon[26606]: Standby manager daemon y restarted 2024-04-22T15:18:32.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:32 smithi145 ceph-mon[26606]: Standby manager daemon y started 2024-04-22T15:18:32.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:32 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.145:0/2165963353' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-22T15:18:32.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:32 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.145:0/2165963353' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:18:32.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:32 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.145:0/2165963353' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-22T15:18:32.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:32 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.145:0/2165963353' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:18:33.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:33 smithi172 ceph-mon[22747]: mgrmap e26: x(active, since 7s), standbys: y 2024-04-22T15:18:33.520 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:33 smithi145 ceph-mon[21899]: mgrmap e26: x(active, since 7s), standbys: y 2024-04-22T15:18:33.521 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:33 smithi145 ceph-mon[26606]: mgrmap e26: x(active, since 7s), standbys: y 2024-04-22T15:18:33.521 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:33 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=error ts=2024-04-22T15:18:33.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.145:8443//api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-04-22T15:18:33.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:33 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:18:33.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.172:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.172 because it doesn't contain any IP SANs" 2024-04-22T15:18:33.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:33 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=warn ts=2024-04-22T15:18:33.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.145:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.145 because it doesn't contain any IP SANs" 2024-04-22T15:18:34.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:34 smithi172 ceph-mon[22747]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-22T15:18:34.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:34 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:34.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:34 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:34.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:34 smithi145 ceph-mon[21899]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-22T15:18:34.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:34 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:34.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:34 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:34.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:34 smithi145 ceph-mon[26606]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-22T15:18:34.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:34 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:34.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:34 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:35.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:35 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:35.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:35 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:35.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:35 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:35.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:35 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:35.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:35 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:35.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:35 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:35.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:35 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:35.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:35 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:35.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:35 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:35.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:35 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:35.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:35 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:35.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:35 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:35.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:35 smithi172 ceph-mon[22747]: Adjusting osd_memory_target on smithi145 to 2992M 2024-04-22T15:18:35.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:35 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:35.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:35 smithi172 ceph-mon[22747]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-22T15:18:36.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:35 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:36.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:35 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:36.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:35 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:36.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:35 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:36.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:35 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:36.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:35 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:36.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:35 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:36.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:35 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:36.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:35 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:36.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:35 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:36.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:35 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:36.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:35 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:36.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:35 smithi145 ceph-mon[21899]: Adjusting osd_memory_target on smithi145 to 2992M 2024-04-22T15:18:36.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:35 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:36.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:35 smithi145 ceph-mon[21899]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-22T15:18:36.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:35 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:36.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:35 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:36.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:35 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:36.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:35 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:36.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:35 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:36.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:35 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:36.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:35 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:36.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:35 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:36.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:35 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:36.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:35 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:36.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:35 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:36.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:35 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:36.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:35 smithi145 ceph-mon[26606]: Adjusting osd_memory_target on smithi145 to 2992M 2024-04-22T15:18:36.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:35 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:36.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:35 smithi145 ceph-mon[26606]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-22T15:18:36.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:36 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:36.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:36 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:36.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:36 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:36.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:36 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:36.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:36 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:36.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:36 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:36.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:36 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:36.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:36 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:36.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:36 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:36.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:36 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:36.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:36 smithi172 ceph-mon[22747]: Adjusting osd_memory_target on smithi172 to 3248M 2024-04-22T15:18:36.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:36 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:36.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:36 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:18:36.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:36 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:18:36.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:36 smithi172 ceph-mon[22747]: Updating smithi145:/etc/ceph/ceph.conf 2024-04-22T15:18:36.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:36 smithi172 ceph-mon[22747]: Updating smithi172:/etc/ceph/ceph.conf 2024-04-22T15:18:36.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:36 smithi172 ceph-mon[22747]: Updating smithi145:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:18:36.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:36 smithi172 ceph-mon[22747]: Updating smithi172:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:18:36.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:36 smithi172 ceph-mon[22747]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:18:36.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:36 smithi172 ceph-mon[22747]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:18:37.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:37.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:37.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:37.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:37.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:37.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:37.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:37.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:37.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:37.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:37.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[21899]: Adjusting osd_memory_target on smithi172 to 3248M 2024-04-22T15:18:37.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:37.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:18:37.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:18:37.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[21899]: Updating smithi145:/etc/ceph/ceph.conf 2024-04-22T15:18:37.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[21899]: Updating smithi172:/etc/ceph/ceph.conf 2024-04-22T15:18:37.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[21899]: Updating smithi145:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:18:37.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[21899]: Updating smithi172:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:18:37.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[21899]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:18:37.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[21899]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:18:37.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:37.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:37.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:37.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:37.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:37.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:37.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:37.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:37.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:37.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-22T15:18:37.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[26606]: Adjusting osd_memory_target on smithi172 to 3248M 2024-04-22T15:18:37.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:37.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:18:37.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:18:37.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[26606]: Updating smithi145:/etc/ceph/ceph.conf 2024-04-22T15:18:37.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[26606]: Updating smithi172:/etc/ceph/ceph.conf 2024-04-22T15:18:37.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[26606]: Updating smithi145:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:18:37.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[26606]: Updating smithi172:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:18:37.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[26606]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:18:37.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:36 smithi145 ceph-mon[26606]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:18:37.791 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:37 smithi145 systemd[1]: Stopping Ceph node-exporter.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:18:37.791 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:37 smithi145 bash[55790]: Error: no container with name or ID "ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter.a" found: no such container 2024-04-22T15:18:37.791 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:37 smithi145 podman[55803]: 2024-04-22 15:18:37.549861258 +0000 UTC m=+0.138117071 container died 5e2f432a509124c22637233b31a0632ea5d5942499c4a315e8159917d07bcc21 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T15:18:38.122 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:37 smithi145 podman[55803]: 2024-04-22 15:18:37.939709002 +0000 UTC m=+0.527964822 container cleanup 5e2f432a509124c22637233b31a0632ea5d5942499c4a315e8159917d07bcc21 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T15:18:38.122 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:37 smithi145 bash[55803]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a 2024-04-22T15:18:38.122 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:37 smithi145 bash[55829]: Error: no container with name or ID "ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter.a" found: no such container 2024-04-22T15:18:38.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:37 smithi145 ceph-mon[21899]: Updating smithi145:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:18:38.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:37 smithi145 ceph-mon[21899]: Updating smithi172:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:18:38.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:37 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:38.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:37 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:38.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:37 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:38.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:37 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:38.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:37 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:38.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:37 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:38.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:37 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:38.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:37 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:38.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:37 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:38.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:37 smithi145 ceph-mon[21899]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-22T15:18:38.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:37 smithi145 ceph-mon[21899]: Reconfiguring daemon node-exporter.a on smithi145 2024-04-22T15:18:38.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:37 smithi145 ceph-mon[21899]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T15:18:38.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:37 smithi145 ceph-mon[26606]: Updating smithi145:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:18:38.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:37 smithi145 ceph-mon[26606]: Updating smithi172:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:18:38.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:37 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:38.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:37 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:38.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:37 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:38.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:37 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:38.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:37 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:38.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:37 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:38.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:37 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:38.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:37 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:38.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:37 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:38.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:37 smithi145 ceph-mon[26606]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-22T15:18:38.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:37 smithi145 ceph-mon[26606]: Reconfiguring daemon node-exporter.a on smithi145 2024-04-22T15:18:38.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:37 smithi145 ceph-mon[26606]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T15:18:38.217 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:37 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:18:37] "GET /metrics HTTP/1.1" 200 35108 "" "Prometheus/2.33.4" 2024-04-22T15:18:38.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:37 smithi172 ceph-mon[22747]: Updating smithi145:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:18:38.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:37 smithi172 ceph-mon[22747]: Updating smithi172:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:18:38.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:37 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:38.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:37 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:38.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:37 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:38.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:37 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:38.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:37 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:38.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:37 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:38.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:37 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:38.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:37 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:38.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:37 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:38.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:37 smithi172 ceph-mon[22747]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-22T15:18:38.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:37 smithi172 ceph-mon[22747]: Reconfiguring daemon node-exporter.a on smithi145 2024-04-22T15:18:38.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:37 smithi172 ceph-mon[22747]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T15:18:38.528 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 podman[55817]: 2024-04-22 15:18:38.182762437 +0000 UTC m=+0.633378962 container remove 5e2f432a509124c22637233b31a0632ea5d5942499c4a315e8159917d07bcc21 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T15:18:38.528 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-22T15:18:38.528 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-22T15:18:38.528 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 systemd[1]: Stopped Ceph node-exporter.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:18:38.528 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@node-exporter.a.service: Consumed 3.778s CPU time. 2024-04-22T15:18:38.528 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 systemd[1]: Starting Ceph node-exporter.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:18:38.528 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 podman[55897]: 2024-04-22 15:18:38.313300942 +0000 UTC m=+0.017352604 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-04-22T15:18:38.529 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 podman[55897]: 2024-04-22 15:18:38.41713849 +0000 UTC m=+0.121190162 container create 7a7a9854f2474fb05f9a17fa7117a367f42f783bb1cb3b6d4ff4c62a8c8b679e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T15:18:38.873 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 podman[55897]: 2024-04-22 15:18:38.61088193 +0000 UTC m=+0.314933559 container init 7a7a9854f2474fb05f9a17fa7117a367f42f783bb1cb3b6d4ff4c62a8c8b679e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T15:18:38.873 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 podman[55897]: 2024-04-22 15:18:38.613637061 +0000 UTC m=+0.317688694 container start 7a7a9854f2474fb05f9a17fa7117a367f42f783bb1cb3b6d4ff4c62a8c8b679e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T15:18:38.873 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.616Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-04-22T15:18:38.873 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.616Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-04-22T15:18:38.873 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.617Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-04-22T15:18:38.873 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.617Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-22T15:18:38.873 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-04-22T15:18:38.873 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=arp 2024-04-22T15:18:38.873 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=bcache 2024-04-22T15:18:38.873 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=bonding 2024-04-22T15:18:38.874 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=btrfs 2024-04-22T15:18:38.874 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=conntrack 2024-04-22T15:18:38.874 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=cpu 2024-04-22T15:18:38.874 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-04-22T15:18:38.874 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=diskstats 2024-04-22T15:18:38.874 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=dmi 2024-04-22T15:18:38.874 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=edac 2024-04-22T15:18:38.874 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=entropy 2024-04-22T15:18:38.875 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-04-22T15:18:38.875 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=filefd 2024-04-22T15:18:38.875 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=filesystem 2024-04-22T15:18:38.875 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=hwmon 2024-04-22T15:18:38.875 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=infiniband 2024-04-22T15:18:38.875 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=ipvs 2024-04-22T15:18:38.875 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=loadavg 2024-04-22T15:18:38.875 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=mdadm 2024-04-22T15:18:38.875 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=meminfo 2024-04-22T15:18:38.875 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=netclass 2024-04-22T15:18:38.875 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=netdev 2024-04-22T15:18:38.875 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=netstat 2024-04-22T15:18:38.875 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=nfs 2024-04-22T15:18:38.875 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=nfsd 2024-04-22T15:18:38.875 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=nvme 2024-04-22T15:18:38.876 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=os 2024-04-22T15:18:38.876 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-04-22T15:18:38.876 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=pressure 2024-04-22T15:18:38.876 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=rapl 2024-04-22T15:18:38.876 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=schedstat 2024-04-22T15:18:38.876 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=sockstat 2024-04-22T15:18:38.876 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=softnet 2024-04-22T15:18:38.876 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=stat 2024-04-22T15:18:38.876 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=tapestats 2024-04-22T15:18:38.876 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=textfile 2024-04-22T15:18:38.876 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-04-22T15:18:38.876 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=time 2024-04-22T15:18:38.876 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-04-22T15:18:38.876 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=uname 2024-04-22T15:18:38.876 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=vmstat 2024-04-22T15:18:38.876 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=xfs 2024-04-22T15:18:38.877 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:115 level=info collector=zfs 2024-04-22T15:18:38.877 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-04-22T15:18:38.877 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[55911]: ts=2024-04-22T15:18:38.618Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-04-22T15:18:38.877 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 bash[55897]: 7a7a9854f2474fb05f9a17fa7117a367f42f783bb1cb3b6d4ff4c62a8c8b679e 2024-04-22T15:18:38.877 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:18:38 smithi145 systemd[1]: Started Ceph node-exporter.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:18:40.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:39 smithi145 ceph-mon[21899]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:18:40.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:39 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:40.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:39 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:40.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:39 smithi145 ceph-mon[21899]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-22T15:18:40.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:39 smithi145 ceph-mon[21899]: Reconfiguring daemon alertmanager.a on smithi145 2024-04-22T15:18:40.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:39 smithi145 ceph-mon[21899]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T15:18:40.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:39 smithi145 ceph-mon[26606]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:18:40.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:39 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:40.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:39 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:40.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:39 smithi145 ceph-mon[26606]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-22T15:18:40.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:39 smithi145 ceph-mon[26606]: Reconfiguring daemon alertmanager.a on smithi145 2024-04-22T15:18:40.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:39 smithi145 ceph-mon[26606]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T15:18:40.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:39 smithi172 ceph-mon[22747]: from='client.14805 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:18:40.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:39 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:40.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:39 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:40.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:39 smithi172 ceph-mon[22747]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-22T15:18:40.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:39 smithi172 ceph-mon[22747]: Reconfiguring daemon alertmanager.a on smithi145 2024-04-22T15:18:40.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:39 smithi172 ceph-mon[22747]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T15:18:41.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:41 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:18:41.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:41 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:18:41.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:41 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:18:41.938 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:18:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[22131]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:18:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:18:42.226 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:42 smithi145 ceph-mon[21899]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T15:18:42.226 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:42 smithi145 ceph-mon[26606]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T15:18:42.227 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:42 smithi145 systemd[1]: Stopping Ceph alertmanager.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:18:42.227 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:42 smithi145 bash[56082]: Error: no container with name or ID "ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager.a" found: no such container 2024-04-22T15:18:42.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:42 smithi172 ceph-mon[22747]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T15:18:42.468 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:42 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:18:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: |\n Pool '{{ $labels.name }}' will be full in less than 5 days assuming the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed it's capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"172.21.15.172:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:18:42.622 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:42 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[46705]: level=info ts=2024-04-22T15:18:42.223Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-04-22T15:18:42.622 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:42 smithi145 podman[56093]: 2024-04-22 15:18:42.264889561 +0000 UTC m=+0.141175820 container died 9cbf618f121210121a6236180d6f9acf72362c6c023ade92234d7495e5d5fb91 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T15:18:42.936 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:42 smithi145 podman[56093]: 2024-04-22 15:18:42.641261258 +0000 UTC m=+0.517547524 container cleanup 9cbf618f121210121a6236180d6f9acf72362c6c023ade92234d7495e5d5fb91 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T15:18:42.936 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:42 smithi145 bash[56093]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a 2024-04-22T15:18:42.936 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:42 smithi145 bash[56119]: Error: no container with name or ID "ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager.a" found: no such container 2024-04-22T15:18:43.302 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:42 smithi145 podman[56107]: 2024-04-22 15:18:42.934116265 +0000 UTC m=+0.663682862 container remove 9cbf618f121210121a6236180d6f9acf72362c6c023ade92234d7495e5d5fb91 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T15:18:43.302 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:43 smithi145 podman[56107]: 2024-04-22 15:18:43.067952396 +0000 UTC m=+0.797518947 volume remove 2a003ea9672060f2d287c43633e2d33b4877f2624de207e76dfabf83f212d7f2 2024-04-22T15:18:43.302 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:43 smithi145 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-22T15:18:43.302 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:43 smithi145 systemd[1]: Stopped Ceph alertmanager.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:18:43.302 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:43 smithi145 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@alertmanager.a.service: Consumed 1.054s CPU time. 2024-04-22T15:18:43.302 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:43 smithi145 systemd[1]: Starting Ceph alertmanager.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:18:43.622 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:43 smithi145 podman[56192]: 2024-04-22 15:18:43.20322101 +0000 UTC m=+0.018227771 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-04-22T15:18:43.622 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:43 smithi145 podman[56192]: 2024-04-22 15:18:43.327689141 +0000 UTC m=+0.142695939 volume create b7895904af97554f62e01b94b5990cdffae4f0ef93cb98e2fed4b6aa4d96e495 2024-04-22T15:18:43.622 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:43 smithi145 podman[56192]: 2024-04-22 15:18:43.42799791 +0000 UTC m=+0.243004665 container create 2b762f668fa703842a8b08ef93b0a393886d5d72787fe3259e3b56c26eca2399 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T15:18:44.122 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:43 smithi145 podman[56192]: 2024-04-22 15:18:43.772636584 +0000 UTC m=+0.587643337 container init 2b762f668fa703842a8b08ef93b0a393886d5d72787fe3259e3b56c26eca2399 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T15:18:44.123 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:43 smithi145 podman[56192]: 2024-04-22 15:18:43.775373866 +0000 UTC m=+0.590380628 container start 2b762f668fa703842a8b08ef93b0a393886d5d72787fe3259e3b56c26eca2399 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T15:18:44.123 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[56207]: level=info ts=2024-04-22T15:18:43.786Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-04-22T15:18:44.123 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[56207]: level=info ts=2024-04-22T15:18:43.786Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-04-22T15:18:44.123 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[56207]: level=info ts=2024-04-22T15:18:43.787Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.145 port=9094 2024-04-22T15:18:44.123 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[56207]: level=info ts=2024-04-22T15:18:43.789Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-22T15:18:44.123 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[56207]: level=info ts=2024-04-22T15:18:43.814Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-22T15:18:44.123 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[56207]: level=info ts=2024-04-22T15:18:43.815Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-22T15:18:44.123 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[56207]: level=info ts=2024-04-22T15:18:43.817Z caller=main.go:518 msg=Listening address=:9093 2024-04-22T15:18:44.123 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[56207]: level=info ts=2024-04-22T15:18:43.817Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-04-22T15:18:44.123 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:43 smithi145 bash[56192]: 2b762f668fa703842a8b08ef93b0a393886d5d72787fe3259e3b56c26eca2399 2024-04-22T15:18:44.123 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:43 smithi145 systemd[1]: Started Ceph alertmanager.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:18:44.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:44 smithi172 ceph-mon[22747]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T15:18:44.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:44 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:44.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:44 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:44.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:44 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:18:44.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:44 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:18:44.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:44 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:18:44.514 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:44 smithi145 ceph-mon[21899]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T15:18:44.515 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:44 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:44.515 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:44 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:44.515 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:44 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:18:44.515 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:44 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:18:44.515 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:44 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:18:44.515 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:44 smithi145 ceph-mon[26606]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T15:18:44.515 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:44 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:44.515 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:44 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:44.515 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:44 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:18:44.515 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:44 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:18:44.515 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:44 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:18:45.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:45 smithi145 ceph-mon[21899]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:18:45.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:45 smithi145 ceph-mon[21899]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:18:45.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:45 smithi145 ceph-mon[26606]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:18:45.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:45 smithi145 ceph-mon[26606]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:18:45.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:45 smithi172 ceph-mon[22747]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:18:45.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:45 smithi172 ceph-mon[22747]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:18:46.058 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[56207]: level=info ts=2024-04-22T15:18:45.790Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000947165s 2024-04-22T15:18:46.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:46 smithi145 ceph-mon[21899]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:18:46.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:46 smithi145 ceph-mon[26606]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:18:46.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:46 smithi172 ceph-mon[22747]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:18:47.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:18:47.016+0000 7fd668b72700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:18:47.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Creating ceph-iscsi config... 2024-04-22T15:18:47.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:18:47.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:18:47.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:18:47.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:18:47.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:18:47.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Traceback (most recent call last): 2024-04-22T15:18:47.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:18:47.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: return _run_code(code, main_globals, None, 2024-04-22T15:18:47.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:18:47.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: exec(code, run_globals) 2024-04-22T15:18:47.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:18:47.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:18:47.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:18:47.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:18:47.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:18:47.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:18:47.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:18:47.470 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:18:47.470 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:18:47.470 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Traceback (most recent call last): 2024-04-22T15:18:47.470 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:18:47.470 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:18:47.470 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:18:47.470 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:18:47.470 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:18:47.471 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:18:47.471 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:18:47.471 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: return future.result(timeout) 2024-04-22T15:18:47.471 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:18:47.471 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: return self.__get_result() 2024-04-22T15:18:47.471 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:18:47.471 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: raise self._exception 2024-04-22T15:18:47.471 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:18:47.471 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: use_current_daemon_image=reconfig, 2024-04-22T15:18:47.471 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:18:47.471 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:18:47.471 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:18:47.471 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Creating ceph-iscsi config... 2024-04-22T15:18:47.471 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:18:47.471 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:18:47.472 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:18:47.472 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:18:47.472 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:18:47.472 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Traceback (most recent call last): 2024-04-22T15:18:47.472 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:18:47.472 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: return _run_code(code, main_globals, None, 2024-04-22T15:18:47.472 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:18:47.472 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: exec(code, run_globals) 2024-04-22T15:18:47.472 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:18:47.472 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:18:47.472 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:18:47.473 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:18:47.473 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:18:47.473 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:18:47.473 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:18:47.473 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:18:47.473 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:18:48.085 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:18:47] "GET /metrics HTTP/1.1" 200 37905 "" "Prometheus/2.33.4" 2024-04-22T15:18:48.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:48 smithi172 ceph-mon[22747]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:18:48.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:18:48.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:18:48.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:18:48.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:18:48.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:18:48.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:18:48.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:18:48.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:18:48.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:18:48.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:18:48.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:18:48.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:18:48.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:18:48.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:18:48.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:18:48.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:18:48.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:18:48.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:18:48.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:18:48.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:18:48.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:18:48.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:18:48.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:18:48.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:18:48.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:18:48.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:18:48.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:18:48.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:18:48.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:18:48.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:18:48.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:18:48.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:18:48.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:18:48.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:18:48.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:18:48.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:18:48.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:18:48.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:18:48.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:18:48.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:18:48.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:18:48.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:18:48.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:18:48.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:18:48.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:18:48.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:18:48.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:18:48.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:18:48.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:18:48.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:18:48.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:18:48.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:18:48.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:18:48.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:18:48.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:18:48.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:18:48.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:18:48.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:18:48.472 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:48 smithi172 ceph-mon[22747]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T15:18:48.472 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:48 smithi172 ceph-mon[22747]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:18:48.472 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:48 smithi172 ceph-mon[22747]: Reconfiguring daemon prometheus.a on smithi172 2024-04-22T15:18:48.524 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:48 smithi145 ceph-mon[26606]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:18:48.524 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:18:48.524 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:18:48.524 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:18:48.525 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:18:48.525 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:18:48.525 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:18:48.525 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:18:48.525 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:18:48.525 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:18:48.525 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:18:48.525 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:18:48.525 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:18:48.525 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:18:48.525 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:18:48.525 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:18:48.525 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:18:48.526 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:18:48.526 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:18:48.526 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:18:48.526 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:18:48.526 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:18:48.526 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:18:48.526 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:18:48.526 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:18:48.526 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:18:48.526 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:18:48.526 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:18:48.526 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:18:48.526 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:18:48.527 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:18:48.527 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:18:48.527 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:18:48.527 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:18:48.527 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:18:48.527 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:18:48.527 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:18:48.527 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:18:48.527 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:18:48.527 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:18:48.527 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:18:48.527 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:18:48.527 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:18:48.527 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:18:48.527 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:18:48.528 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:18:48.528 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:18:48.528 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:18:48.528 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:18:48.528 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:18:48.528 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:18:48.528 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:18:48.528 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:18:48.528 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:18:48.528 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:48 smithi145 ceph-mon[21899]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:18:48.528 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:18:48.529 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:18:48.529 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:18:48.529 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:18:48.529 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:18:48.529 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:18:48.529 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:18:48.529 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:18:48.529 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:18:48.529 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:18:48.529 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:18:48.529 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:18:48.529 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:18:48.529 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:18:48.529 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:18:48.530 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:18:48.530 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:18:48.530 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:18:48.530 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:18:48.530 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:18:48.530 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:18:48.530 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:18:48.530 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:18:48.530 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:18:48.530 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:18:48.530 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:18:48.530 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:18:48.530 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:18:48.531 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:18:48.531 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:18:48.531 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:18:48.531 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:18:48.531 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:18:48.531 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:18:48.531 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:18:48.531 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:18:48.531 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:18:48.531 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:18:48.531 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:18:48.531 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:18:48.531 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:18:48.531 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:18:48.531 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:18:48.531 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:18:48.532 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:18:48.532 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:18:48.532 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:18:48.532 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:18:48.532 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:18:48.532 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:18:48.532 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:18:48.532 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:18:48.532 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:18:48.532 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:18:48.532 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:18:48.532 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:18:48.532 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:18:48.533 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:18:48.533 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:48 smithi145 ceph-mon[26606]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T15:18:48.533 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:48 smithi145 ceph-mon[26606]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:18:48.533 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:48 smithi145 ceph-mon[26606]: Reconfiguring daemon prometheus.a on smithi172 2024-04-22T15:18:48.533 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:18:48.533 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:18:48.533 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:18:48.533 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:18:48.533 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:18:48.533 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:48 smithi145 ceph-mon[21899]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T15:18:48.533 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:48 smithi145 ceph-mon[21899]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:18:48.533 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:48 smithi145 ceph-mon[21899]: Reconfiguring daemon prometheus.a on smithi172 2024-04-22T15:18:50.307 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:50 smithi172 ceph-mon[22747]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T15:18:50.594 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:50 smithi172 systemd[1]: Stopping Ceph prometheus.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:18:50.595 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:50 smithi172 bash[46031]: Error: no container with name or ID "ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-22T15:18:50.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:50 smithi145 ceph-mon[21899]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T15:18:50.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:50 smithi145 ceph-mon[26606]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T15:18:50.854 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:50 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:18:50.591Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-22T15:18:50.854 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:50 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:18:50.592Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-22T15:18:50.854 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:50 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:18:50.592Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-22T15:18:50.854 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:50 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:18:50.592Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-22T15:18:50.854 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:50 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:18:50.592Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-22T15:18:50.854 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:50 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:18:50.592Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-22T15:18:50.855 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:50 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:18:50.592Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-22T15:18:50.855 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:50 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:18:50.592Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-22T15:18:50.855 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:50 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:18:50.592Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-22T15:18:50.855 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:50 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:18:50.592Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-22T15:18:50.855 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:50 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:18:50.592Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-22T15:18:50.855 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:50 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[39672]: ts=2024-04-22T15:18:50.592Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-22T15:18:50.855 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:50 smithi172 podman[46044]: 2024-04-22 15:18:50.6020586 +0000 UTC m=+0.126914927 container died 9121b6b1d1de107a43e0fdb307f670fb4116add9e9e4fd48bb09e8c4a4f93e8f (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:18:51.104 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:50 smithi172 podman[46044]: 2024-04-22 15:18:50.942314834 +0000 UTC m=+0.467171170 container cleanup 9121b6b1d1de107a43e0fdb307f670fb4116add9e9e4fd48bb09e8c4a4f93e8f (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:18:51.104 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:50 smithi172 bash[46044]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a 2024-04-22T15:18:51.383 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:51 smithi172 podman[46058]: 2024-04-22 15:18:51.101957191 +0000 UTC m=+0.501322024 container remove 9121b6b1d1de107a43e0fdb307f670fb4116add9e9e4fd48bb09e8c4a4f93e8f (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:18:51.383 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:51 smithi172 bash[46071]: Error: no container with name or ID "ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-22T15:18:51.383 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:51 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-22T15:18:51.383 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:51 smithi172 systemd[1]: Stopped Ceph prometheus.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:18:51.383 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:51 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.965s CPU time. 2024-04-22T15:18:51.383 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:51 smithi172 systemd[1]: Starting Ceph prometheus.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:18:51.665 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:51 smithi172 podman[46137]: 2024-04-22 15:18:51.28510351 +0000 UTC m=+0.018746163 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-22T15:18:51.665 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:51 smithi172 podman[46137]: 2024-04-22 15:18:51.386235111 +0000 UTC m=+0.119877767 container create b46c16fe4d433151f713ecc1c3b13325a9b66dd6d9368271578f55acc1477ead (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:18:51.666 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:51 smithi172 podman[46137]: 2024-04-22 15:18:51.588227249 +0000 UTC m=+0.321869903 container init b46c16fe4d433151f713ecc1c3b13325a9b66dd6d9368271578f55acc1477ead (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:18:51.666 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:51 smithi172 podman[46137]: 2024-04-22 15:18:51.590717839 +0000 UTC m=+0.324360493 container start b46c16fe4d433151f713ecc1c3b13325a9b66dd6d9368271578f55acc1477ead (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:18:51.666 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:51 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:18:51.613Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-22T15:18:51.666 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:51 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:18:51.613Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-22T15:18:51.666 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:51 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:18:51.614Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-22T15:18:51.666 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:51 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:18:51.614Z caller=main.go:518 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi172 (none))" 2024-04-22T15:18:51.666 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:51 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:18:51.614Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-22T15:18:51.666 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:51 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:18:51.614Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-22T15:18:51.666 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:51 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:18:51.615Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-22T15:18:51.666 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:51 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:18:51.616Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-22T15:18:51.666 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:51 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:18:51.617Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-22T15:18:51.666 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:51 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:18:51.619Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-22T15:18:51.666 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:51 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:18:51.619Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.646µs 2024-04-22T15:18:51.666 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:51 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:18:51.619Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-22T15:18:51.667 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:51 smithi172 bash[46137]: b46c16fe4d433151f713ecc1c3b13325a9b66dd6d9368271578f55acc1477ead 2024-04-22T15:18:51.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:51 smithi172 systemd[1]: Started Ceph prometheus.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:18:51.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:51 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:18:51.818Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-04-22T15:18:52.370 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:52 smithi172 systemd[1]: Stopping Ceph node-exporter.b for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:18:52.370 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:52 smithi172 bash[46262]: Error: no container with name or ID "ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter.b" found: no such container 2024-04-22T15:18:52.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:52 smithi172 ceph-mon[22747]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T15:18:52.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:52 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:52.371 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:52 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:52.371 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:52 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:18:52.046Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-04-22T15:18:52.371 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:52 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:18:52.046Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-04-22T15:18:52.371 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:52 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:18:52.046Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=28.33µs wal_replay_duration=427.585345ms total_replay_duration=427.62691ms 2024-04-22T15:18:52.371 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:52 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:18:52.050Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-22T15:18:52.371 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:52 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:18:52.050Z caller=main.go:947 level=info msg="TSDB started" 2024-04-22T15:18:52.371 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:52 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:18:52.050Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-22T15:18:52.371 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:52 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:18:52.073Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=23.126124ms db_storage=1.125µs remote_storage=1.456µs web_handler=315ns query_engine=696ns scrape=411.916µs scrape_sd=90.984µs notify=11.17µs notify_sd=8.761µs rules=22.189869ms 2024-04-22T15:18:52.372 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:18:52 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:18:52.073Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-22T15:18:52.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:52 smithi145 ceph-mon[21899]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T15:18:52.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:52 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:52.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:52 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:52.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:52 smithi145 ceph-mon[26606]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T15:18:52.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:52 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:52.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:52 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:52.653 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:52 smithi172 podman[46275]: 2024-04-22 15:18:52.368201628 +0000 UTC m=+0.114112648 container died 759c7e0c3e41d5ed7ae0c268f94143807f31869084177f7b77c89c31eeaf141d (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T15:18:52.930 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:52 smithi172 podman[46275]: 2024-04-22 15:18:52.724617914 +0000 UTC m=+0.470528941 container cleanup 759c7e0c3e41d5ed7ae0c268f94143807f31869084177f7b77c89c31eeaf141d (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T15:18:52.930 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:52 smithi172 bash[46275]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b 2024-04-22T15:18:52.930 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:52 smithi172 podman[46289]: 2024-04-22 15:18:52.892363288 +0000 UTC m=+0.528466186 container remove 759c7e0c3e41d5ed7ae0c268f94143807f31869084177f7b77c89c31eeaf141d (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T15:18:52.930 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:52 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-22T15:18:53.183 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:52 smithi172 bash[46303]: Error: no container with name or ID "ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter.b" found: no such container 2024-04-22T15:18:53.184 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:52 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-22T15:18:53.184 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:52 smithi172 systemd[1]: Stopped Ceph node-exporter.b for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:18:53.184 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:52 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@node-exporter.b.service: Consumed 3.642s CPU time. 2024-04-22T15:18:53.184 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:52 smithi172 systemd[1]: Starting Ceph node-exporter.b for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:18:53.436 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-mon[22747]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-22T15:18:53.437 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-mon[22747]: Reconfiguring daemon node-exporter.b on smithi172 2024-04-22T15:18:53.437 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 podman[46376]: 2024-04-22 15:18:53.083853639 +0000 UTC m=+0.025146327 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-04-22T15:18:53.437 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 podman[46376]: 2024-04-22 15:18:53.201751325 +0000 UTC m=+0.143044002 container create e5b0a512c8352956d8f1bdfbb7df1bda1d18ba36d77f6db5a5898125246a62ed (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T15:18:53.437 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 podman[46376]: 2024-04-22 15:18:53.429175629 +0000 UTC m=+0.370468323 container init e5b0a512c8352956d8f1bdfbb7df1bda1d18ba36d77f6db5a5898125246a62ed (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T15:18:53.437 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 podman[46376]: 2024-04-22 15:18:53.43177444 +0000 UTC m=+0.373067132 container start e5b0a512c8352956d8f1bdfbb7df1bda1d18ba36d77f6db5a5898125246a62ed (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T15:18:53.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:53 smithi145 ceph-mon[21899]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-22T15:18:53.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:53 smithi145 ceph-mon[21899]: Reconfiguring daemon node-exporter.b on smithi172 2024-04-22T15:18:53.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:53 smithi145 ceph-mon[26606]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-22T15:18:53.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:53 smithi145 ceph-mon[26606]: Reconfiguring daemon node-exporter.b on smithi172 2024-04-22T15:18:53.718 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.434Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-04-22T15:18:53.718 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.434Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-04-22T15:18:53.718 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.435Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-04-22T15:18:53.719 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.435Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-22T15:18:53.719 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.435Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-04-22T15:18:53.719 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.435Z caller=node_exporter.go:115 level=info collector=arp 2024-04-22T15:18:53.719 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.435Z caller=node_exporter.go:115 level=info collector=bcache 2024-04-22T15:18:53.719 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.435Z caller=node_exporter.go:115 level=info collector=bonding 2024-04-22T15:18:53.719 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.435Z caller=node_exporter.go:115 level=info collector=btrfs 2024-04-22T15:18:53.719 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.435Z caller=node_exporter.go:115 level=info collector=conntrack 2024-04-22T15:18:53.719 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.435Z caller=node_exporter.go:115 level=info collector=cpu 2024-04-22T15:18:53.719 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.435Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-04-22T15:18:53.719 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.435Z caller=node_exporter.go:115 level=info collector=diskstats 2024-04-22T15:18:53.719 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.435Z caller=node_exporter.go:115 level=info collector=dmi 2024-04-22T15:18:53.720 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.435Z caller=node_exporter.go:115 level=info collector=edac 2024-04-22T15:18:53.720 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.435Z caller=node_exporter.go:115 level=info collector=entropy 2024-04-22T15:18:53.720 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.435Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-04-22T15:18:53.720 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.435Z caller=node_exporter.go:115 level=info collector=filefd 2024-04-22T15:18:53.720 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.435Z caller=node_exporter.go:115 level=info collector=filesystem 2024-04-22T15:18:53.720 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.435Z caller=node_exporter.go:115 level=info collector=hwmon 2024-04-22T15:18:53.720 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.435Z caller=node_exporter.go:115 level=info collector=infiniband 2024-04-22T15:18:53.721 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.435Z caller=node_exporter.go:115 level=info collector=ipvs 2024-04-22T15:18:53.721 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.435Z caller=node_exporter.go:115 level=info collector=loadavg 2024-04-22T15:18:53.721 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.435Z caller=node_exporter.go:115 level=info collector=mdadm 2024-04-22T15:18:53.721 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.435Z caller=node_exporter.go:115 level=info collector=meminfo 2024-04-22T15:18:53.721 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.435Z caller=node_exporter.go:115 level=info collector=netclass 2024-04-22T15:18:53.721 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.435Z caller=node_exporter.go:115 level=info collector=netdev 2024-04-22T15:18:53.721 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.435Z caller=node_exporter.go:115 level=info collector=netstat 2024-04-22T15:18:53.721 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.435Z caller=node_exporter.go:115 level=info collector=nfs 2024-04-22T15:18:53.721 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.436Z caller=node_exporter.go:115 level=info collector=nfsd 2024-04-22T15:18:53.721 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.436Z caller=node_exporter.go:115 level=info collector=nvme 2024-04-22T15:18:53.721 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.436Z caller=node_exporter.go:115 level=info collector=os 2024-04-22T15:18:53.722 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.436Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-04-22T15:18:53.722 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.436Z caller=node_exporter.go:115 level=info collector=pressure 2024-04-22T15:18:53.722 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.436Z caller=node_exporter.go:115 level=info collector=rapl 2024-04-22T15:18:53.722 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.436Z caller=node_exporter.go:115 level=info collector=schedstat 2024-04-22T15:18:53.722 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.436Z caller=node_exporter.go:115 level=info collector=sockstat 2024-04-22T15:18:53.722 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.436Z caller=node_exporter.go:115 level=info collector=softnet 2024-04-22T15:18:53.722 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.436Z caller=node_exporter.go:115 level=info collector=stat 2024-04-22T15:18:53.722 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.436Z caller=node_exporter.go:115 level=info collector=tapestats 2024-04-22T15:18:53.722 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.436Z caller=node_exporter.go:115 level=info collector=textfile 2024-04-22T15:18:53.722 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.436Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-04-22T15:18:53.722 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.436Z caller=node_exporter.go:115 level=info collector=time 2024-04-22T15:18:53.722 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.436Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-04-22T15:18:53.722 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.436Z caller=node_exporter.go:115 level=info collector=uname 2024-04-22T15:18:53.723 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.436Z caller=node_exporter.go:115 level=info collector=vmstat 2024-04-22T15:18:53.723 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.436Z caller=node_exporter.go:115 level=info collector=xfs 2024-04-22T15:18:53.723 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.436Z caller=node_exporter.go:115 level=info collector=zfs 2024-04-22T15:18:53.723 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.436Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-04-22T15:18:53.723 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[46393]: ts=2024-04-22T15:18:53.436Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-04-22T15:18:53.723 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 bash[46376]: e5b0a512c8352956d8f1bdfbb7df1bda1d18ba36d77f6db5a5898125246a62ed 2024-04-22T15:18:53.723 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:18:53 smithi172 systemd[1]: Started Ceph node-exporter.b for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:18:54.122 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:18:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[56207]: level=info ts=2024-04-22T15:18:53.793Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003741026s 2024-04-22T15:18:54.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:54 smithi172 ceph-mon[22747]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T15:18:54.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:54 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:54.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:54 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:54.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:54 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:54.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:54 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:54.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:54 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T15:18:54.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:54 smithi145 ceph-mon[21899]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T15:18:54.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:54 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:54.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:54 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:54.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:54 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:54.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:54 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:54.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:54 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T15:18:54.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:54 smithi145 ceph-mon[26606]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T15:18:54.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:54 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:54.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:54 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:54.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:54 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:54.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:54 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:54.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:54 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T15:18:55.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:55 smithi172 ceph-mon[22747]: Reconfiguring grafana.a (dependencies changed)... 2024-04-22T15:18:55.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:55 smithi172 ceph-mon[22747]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-22T15:18:55.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:55 smithi172 ceph-mon[22747]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T15:18:55.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:55 smithi172 ceph-mon[22747]: Reconfiguring daemon grafana.a on smithi172 2024-04-22T15:18:55.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:55 smithi145 ceph-mon[21899]: Reconfiguring grafana.a (dependencies changed)... 2024-04-22T15:18:55.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:55 smithi145 ceph-mon[21899]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-22T15:18:55.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:55 smithi145 ceph-mon[21899]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T15:18:55.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:55 smithi145 ceph-mon[21899]: Reconfiguring daemon grafana.a on smithi172 2024-04-22T15:18:55.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:55 smithi145 ceph-mon[26606]: Reconfiguring grafana.a (dependencies changed)... 2024-04-22T15:18:55.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:55 smithi145 ceph-mon[26606]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-22T15:18:55.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:55 smithi145 ceph-mon[26606]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T15:18:55.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:55 smithi145 ceph-mon[26606]: Reconfiguring daemon grafana.a on smithi172 2024-04-22T15:18:56.525 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:56 smithi172 systemd[1]: Stopping Ceph grafana.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:18:56.525 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:56 smithi172 bash[46565]: Error: no container with name or ID "ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana.a" found: no such container 2024-04-22T15:18:56.525 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:56 smithi172 ceph-mon[22747]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:18:56.525 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:56 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:18:56.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:56 smithi145 ceph-mon[26606]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:18:56.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:56 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:18:56.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:56 smithi145 ceph-mon[21899]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:18:56.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:56 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:18:56.824 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:56 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:18:56+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-04-22T15:18:56.825 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:56 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[38872]: t=2024-04-22T15:18:56+0000 lvl=info msg="Database locked, sleeping then retrying" logger=sqlstore error="database is locked" retry=0 2024-04-22T15:18:56.825 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:56 smithi172 podman[46578]: 2024-04-22 15:18:56.542851544 +0000 UTC m=+0.145233405 container died b5aa7bda58720a225de10612e27bfb4c9de1c637292ea33d5545a483eeb764ba (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a, io.buildah.version=1.24.2, release=236.1648460182, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-03-28T10:36:18.413762, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, name=ubi8, description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., io.openshift.expose-services=) 2024-04-22T15:18:57.217 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:56 smithi172 podman[46578]: 2024-04-22 15:18:56.89637077 +0000 UTC m=+0.498752636 container cleanup b5aa7bda58720a225de10612e27bfb4c9de1c637292ea33d5545a483eeb764ba (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a, version=8.5, vcs-type=git, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, description=Ceph Grafana Container, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-03-28T10:36:18.413762, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, vendor=Red Hat, Inc., io.buildah.version=1.24.2, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, release=236.1648460182) 2024-04-22T15:18:57.217 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:56 smithi172 bash[46578]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a 2024-04-22T15:18:57.217 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:56 smithi172 bash[46604]: Error: no container with name or ID "ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana.a" found: no such container 2024-04-22T15:18:57.705 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 podman[46592]: 2024-04-22 15:18:57.364580568 +0000 UTC m=+0.823621181 container remove b5aa7bda58720a225de10612e27bfb4c9de1c637292ea33d5545a483eeb764ba (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.24.2, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, name=ubi8, build-date=2022-03-28T10:36:18.413762, description=Ceph Grafana Container, distribution-scope=public, vcs-type=git, release=236.1648460182) 2024-04-22T15:18:57.705 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@grafana.a.service: Deactivated successfully. 2024-04-22T15:18:57.705 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 systemd[1]: Stopped Ceph grafana.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:18:57.705 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@grafana.a.service: Consumed 3.296s CPU time. 2024-04-22T15:18:57.706 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 systemd[1]: Starting Ceph grafana.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:18:57.706 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 podman[46673]: 2024-04-22 15:18:57.496428625 +0000 UTC m=+0.018020668 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-04-22T15:18:57.706 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 podman[46673]: 2024-04-22 15:18:57.607527961 +0000 UTC m=+0.129120033 container create 20a9f6cd18595d1d691a98eefc0be0d47481ceb7fee46ebeaad70c648d121c6f (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a, vcs-type=git, io.buildah.version=1.24.2, architecture=x86_64, vendor=Red Hat, Inc., description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, release=236.1648460182, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , build-date=2022-03-28T10:36:18.413762) 2024-04-22T15:18:58.055 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:18:57] ENGINE Bus STOPPING 2024-04-22T15:18:58.055 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:18:57] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-22T15:18:58.055 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:18:57] ENGINE Bus STOPPED 2024-04-22T15:18:58.055 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:18:57] ENGINE Bus STARTING 2024-04-22T15:18:58.055 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:18:58] ENGINE Serving on http://:::9283 2024-04-22T15:18:58.055 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:18:58] ENGINE Bus STARTED 2024-04-22T15:18:58.056 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 podman[46673]: 2024-04-22 15:18:57.784753696 +0000 UTC m=+0.306345729 container init 20a9f6cd18595d1d691a98eefc0be0d47481ceb7fee46ebeaad70c648d121c6f (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, io.buildah.version=1.24.2, vcs-type=git, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, release=236.1648460182, build-date=2022-03-28T10:36:18.413762, description=Ceph Grafana Container, io.openshift.tags=base rhel8, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5) 2024-04-22T15:18:58.056 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 podman[46673]: 2024-04-22 15:18:57.787268806 +0000 UTC m=+0.308860837 container start 20a9f6cd18595d1d691a98eefc0be0d47481ceb7fee46ebeaad70c648d121c6f (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a, io.buildah.version=1.24.2, architecture=x86_64, build-date=2022-03-28T10:36:18.413762, description=Ceph Grafana Container, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, release=236.1648460182, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, com.redhat.component=ubi8-container, distribution-scope=public, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=) 2024-04-22T15:18:58.056 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[46687]: t=2024-04-22T15:18:57+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-04-22T15:18:58.056 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[46687]: t=2024-04-22T15:18:57+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-04-22T15:18:58.056 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[46687]: t=2024-04-22T15:18:57+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-04-22T15:18:58.056 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[46687]: t=2024-04-22T15:18:57+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-04-22T15:18:58.056 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[46687]: t=2024-04-22T15:18:57+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-04-22T15:18:58.056 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[46687]: t=2024-04-22T15:18:57+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-04-22T15:18:58.057 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[46687]: t=2024-04-22T15:18:57+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-04-22T15:18:58.057 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[46687]: t=2024-04-22T15:18:57+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-04-22T15:18:58.057 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[46687]: t=2024-04-22T15:18:57+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-04-22T15:18:58.057 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[46687]: t=2024-04-22T15:18:57+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-04-22T15:18:58.057 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[46687]: t=2024-04-22T15:18:57+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-04-22T15:18:58.057 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[46687]: t=2024-04-22T15:18:57+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-04-22T15:18:58.057 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[46687]: t=2024-04-22T15:18:57+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-04-22T15:18:58.057 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[46687]: t=2024-04-22T15:18:57+0000 lvl=info msg="App mode production" logger=settings 2024-04-22T15:18:58.057 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[46687]: t=2024-04-22T15:18:57+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-04-22T15:18:58.057 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[46687]: t=2024-04-22T15:18:57+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-04-22T15:18:58.057 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[46687]: t=2024-04-22T15:18:57+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-04-22T15:18:58.057 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[46687]: t=2024-04-22T15:18:57+0000 lvl=info msg="migrations completed" logger=migrator performed=0 skipped=377 duration=513.742µs 2024-04-22T15:18:58.057 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[46687]: t=2024-04-22T15:18:57+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-04-22T15:18:58.057 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[46687]: t=2024-04-22T15:18:57+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-04-22T15:18:58.057 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[46687]: t=2024-04-22T15:18:57+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-04-22T15:18:58.058 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[46687]: t=2024-04-22T15:18:57+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-04-22T15:18:58.058 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[46687]: t=2024-04-22T15:18:57+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-04-22T15:18:58.058 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 bash[46673]: 20a9f6cd18595d1d691a98eefc0be0d47481ceb7fee46ebeaad70c648d121c6f 2024-04-22T15:18:58.058 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[46687]: t=2024-04-22T15:18:57+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-04-22T15:18:58.058 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 systemd[1]: Started Ceph grafana.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:18:58.058 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[46687]: t=2024-04-22T15:18:57+0000 lvl=info msg="deleted datasource based on configuration" logger=provisioning.datasources name=Dashboard1 2024-04-22T15:18:58.058 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[46687]: t=2024-04-22T15:18:57+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-04-22T15:18:58.058 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[46687]: t=2024-04-22T15:18:57+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Loki uid=P8E80F9AEF21F6940 2024-04-22T15:18:58.058 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[46687]: t=2024-04-22T15:18:57+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-04-22T15:18:58.058 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[46687]: t=2024-04-22T15:18:57+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-04-22T15:18:58.058 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:18:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[46687]: t=2024-04-22T15:18:57+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-04-22T15:18:58.442 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:58 smithi145 ceph-mon[21899]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:18:58.443 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:58 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:58.443 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:58 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:58.443 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:58 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T15:18:58.443 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:58 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi145.front.sepia.ceph.com:9093"}]: dispatch 2024-04-22T15:18:58.443 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:58 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:58.443 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:58 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:18:58.444 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:58 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi172.front.sepia.ceph.com:9095"}]: dispatch 2024-04-22T15:18:58.444 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:58 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:58.444 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:58 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T15:18:58.444 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:58 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi172.front.sepia.ceph.com:3000"}]: dispatch 2024-04-22T15:18:58.444 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:58 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:58.444 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:58 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:18:58.444 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:58 smithi145 ceph-mon[26606]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:18:58.444 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:58 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:58.444 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:58 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:58.444 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:58 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T15:18:58.444 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:58 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi145.front.sepia.ceph.com:9093"}]: dispatch 2024-04-22T15:18:58.445 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:58 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:58.445 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:58 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:18:58.445 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:58 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi172.front.sepia.ceph.com:9095"}]: dispatch 2024-04-22T15:18:58.445 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:58 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:58.445 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:58 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T15:18:58.445 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:58 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi172.front.sepia.ceph.com:3000"}]: dispatch 2024-04-22T15:18:58.445 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:58 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:58.445 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:58 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:18:58.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:58 smithi172 ceph-mon[22747]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:18:58.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:58 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:58.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:58 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:58.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:58 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T15:18:58.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:58 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi145.front.sepia.ceph.com:9093"}]: dispatch 2024-04-22T15:18:58.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:58 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:58.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:58 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:18:58.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:58 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi172.front.sepia.ceph.com:9095"}]: dispatch 2024-04-22T15:18:58.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:58 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:58.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:58 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T15:18:58.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:58 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi172.front.sepia.ceph.com:3000"}]: dispatch 2024-04-22T15:18:58.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:58 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:18:58.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:58 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:18:58.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:18:58] ENGINE Bus STOPPING 2024-04-22T15:18:58.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:18:58] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-22T15:18:58.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:18:58] ENGINE Bus STOPPED 2024-04-22T15:18:58.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:18:58] ENGINE Bus STARTING 2024-04-22T15:18:58.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:18:58] ENGINE Serving on http://:::9283 2024-04-22T15:18:58.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:18:58] ENGINE Bus STARTED 2024-04-22T15:18:58.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:18:58] ENGINE Bus STOPPING 2024-04-22T15:18:58.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:18:58] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-22T15:18:58.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:18:58] ENGINE Bus STOPPED 2024-04-22T15:18:58.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:18:58] ENGINE Bus STARTING 2024-04-22T15:18:58.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:18:58] ENGINE Serving on http://:::9283 2024-04-22T15:18:58.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:18:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:18:58] ENGINE Bus STARTED 2024-04-22T15:18:59.563 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:59 smithi172 ceph-mon[22747]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T15:18:59.563 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:59 smithi172 ceph-mon[22747]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi145.front.sepia.ceph.com:9093"}]: dispatch 2024-04-22T15:18:59.563 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:59 smithi172 ceph-mon[22747]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:18:59.563 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:59 smithi172 ceph-mon[22747]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi172.front.sepia.ceph.com:9095"}]: dispatch 2024-04-22T15:18:59.563 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:59 smithi172 ceph-mon[22747]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T15:18:59.563 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:59 smithi172 ceph-mon[22747]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi172.front.sepia.ceph.com:3000"}]: dispatch 2024-04-22T15:18:59.563 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:59 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/1705031136' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T15:18:59.563 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:18:59 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2129426931' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1768331515"}]: dispatch 2024-04-22T15:18:59.580 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:59 smithi145 ceph-mon[21899]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T15:18:59.581 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:59 smithi145 ceph-mon[21899]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi145.front.sepia.ceph.com:9093"}]: dispatch 2024-04-22T15:18:59.581 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:59 smithi145 ceph-mon[21899]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:18:59.581 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:59 smithi145 ceph-mon[21899]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi172.front.sepia.ceph.com:9095"}]: dispatch 2024-04-22T15:18:59.581 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:59 smithi145 ceph-mon[21899]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T15:18:59.581 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:59 smithi145 ceph-mon[21899]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi172.front.sepia.ceph.com:3000"}]: dispatch 2024-04-22T15:18:59.581 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:59 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/1705031136' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T15:18:59.581 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:18:59 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2129426931' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1768331515"}]: dispatch 2024-04-22T15:18:59.581 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:59 smithi145 ceph-mon[26606]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T15:18:59.581 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:59 smithi145 ceph-mon[26606]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi145.front.sepia.ceph.com:9093"}]: dispatch 2024-04-22T15:18:59.581 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:59 smithi145 ceph-mon[26606]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:18:59.581 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:59 smithi145 ceph-mon[26606]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi172.front.sepia.ceph.com:9095"}]: dispatch 2024-04-22T15:18:59.581 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:59 smithi145 ceph-mon[26606]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T15:18:59.581 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:59 smithi145 ceph-mon[26606]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi172.front.sepia.ceph.com:3000"}]: dispatch 2024-04-22T15:18:59.581 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:59 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/1705031136' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T15:18:59.582 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:18:59 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2129426931' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1768331515"}]: dispatch 2024-04-22T15:19:00.600 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:00 smithi145 ceph-mon[26606]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:19:00.600 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:00 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2129426931' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1768331515"}]': finished 2024-04-22T15:19:00.600 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:00 smithi145 ceph-mon[26606]: osdmap e98: 8 total, 8 up, 8 in 2024-04-22T15:19:00.600 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:00 smithi145 ceph-mon[21899]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:19:00.600 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:00 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2129426931' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1768331515"}]': finished 2024-04-22T15:19:00.600 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:00 smithi145 ceph-mon[21899]: osdmap e98: 8 total, 8 up, 8 in 2024-04-22T15:19:00.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:00 smithi172 ceph-mon[22747]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:19:00.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:00 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2129426931' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1768331515"}]': finished 2024-04-22T15:19:00.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:00 smithi172 ceph-mon[22747]: osdmap e98: 8 total, 8 up, 8 in 2024-04-22T15:19:01.489 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:01 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/892492130' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/3013561970"}]: dispatch 2024-04-22T15:19:01.489 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:01 smithi145 ceph-mon[26606]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/3013561970"}]: dispatch 2024-04-22T15:19:01.489 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:01 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/892492130' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/3013561970"}]: dispatch 2024-04-22T15:19:01.489 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:01 smithi145 ceph-mon[21899]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/3013561970"}]: dispatch 2024-04-22T15:19:01.602 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:01 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/892492130' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/3013561970"}]: dispatch 2024-04-22T15:19:01.603 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:01 smithi172 ceph-mon[22747]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/3013561970"}]: dispatch 2024-04-22T15:19:01.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:19:01] "GET /metrics HTTP/1.1" 200 37906 "" "Prometheus/2.33.4" 2024-04-22T15:19:02.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:02 smithi145 ceph-mon[21899]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 307 B/s rd, 0 op/s 2024-04-22T15:19:02.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:02 smithi145 ceph-mon[21899]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/3013561970"}]': finished 2024-04-22T15:19:02.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:02 smithi145 ceph-mon[21899]: osdmap e99: 8 total, 8 up, 8 in 2024-04-22T15:19:02.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:02 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/4054797313' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3433168494"}]: dispatch 2024-04-22T15:19:02.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:02 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:19:02.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:02 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:19:02.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:02 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:19:02.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:02 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:19:02.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:02 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:19:02.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:02 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:19:02.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:02 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:19:02.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:02 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:19:02.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:02 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:19:02.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:02 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:19:02.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:02 smithi145 ceph-mon[26606]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 307 B/s rd, 0 op/s 2024-04-22T15:19:02.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:02 smithi145 ceph-mon[26606]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/3013561970"}]': finished 2024-04-22T15:19:02.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:02 smithi145 ceph-mon[26606]: osdmap e99: 8 total, 8 up, 8 in 2024-04-22T15:19:02.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:02 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/4054797313' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3433168494"}]: dispatch 2024-04-22T15:19:02.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:02 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:19:02.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:02 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:19:02.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:02 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:19:02.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:02 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:19:02.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:02 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:19:02.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:02 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:19:02.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:02 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:19:02.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:02 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:19:02.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:02 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:19:02.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:02 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:19:02.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:02 smithi172 ceph-mon[22747]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 307 B/s rd, 0 op/s 2024-04-22T15:19:02.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:02 smithi172 ceph-mon[22747]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/3013561970"}]': finished 2024-04-22T15:19:02.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:02 smithi172 ceph-mon[22747]: osdmap e99: 8 total, 8 up, 8 in 2024-04-22T15:19:02.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:02 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/4054797313' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3433168494"}]: dispatch 2024-04-22T15:19:02.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:02 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:19:02.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:02 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:19:02.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:02 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:19:02.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:02 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:19:02.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:02 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:19:02.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:02 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:19:02.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:02 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:19:02.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:02 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:19:02.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:02 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:19:02.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:02 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:19:03.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:03 smithi145 ceph-mon[21899]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:19:03.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:03 smithi145 ceph-mon[21899]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:19:03.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:03 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/4054797313' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3433168494"}]': finished 2024-04-22T15:19:03.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:03 smithi145 ceph-mon[21899]: osdmap e100: 8 total, 8 up, 8 in 2024-04-22T15:19:03.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:03 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/924124128' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/3013561970"}]: dispatch 2024-04-22T15:19:03.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:03 smithi145 ceph-mon[21899]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/3013561970"}]: dispatch 2024-04-22T15:19:03.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:03 smithi145 ceph-mon[26606]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:19:03.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:03 smithi145 ceph-mon[26606]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:19:03.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:03 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/4054797313' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3433168494"}]': finished 2024-04-22T15:19:03.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:03 smithi145 ceph-mon[26606]: osdmap e100: 8 total, 8 up, 8 in 2024-04-22T15:19:03.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:03 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/924124128' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/3013561970"}]: dispatch 2024-04-22T15:19:03.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:03 smithi145 ceph-mon[26606]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/3013561970"}]: dispatch 2024-04-22T15:19:03.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:03 smithi172 ceph-mon[22747]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:19:03.767 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:03 smithi172 ceph-mon[22747]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:19:03.767 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:03 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/4054797313' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3433168494"}]': finished 2024-04-22T15:19:03.767 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:03 smithi172 ceph-mon[22747]: osdmap e100: 8 total, 8 up, 8 in 2024-04-22T15:19:03.767 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:03 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/924124128' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/3013561970"}]: dispatch 2024-04-22T15:19:03.767 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:03 smithi172 ceph-mon[22747]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/3013561970"}]: dispatch 2024-04-22T15:19:04.521 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:04 smithi145 ceph-mon[21899]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:19:04.521 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:04 smithi145 ceph-mon[21899]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/3013561970"}]': finished 2024-04-22T15:19:04.521 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:04 smithi145 ceph-mon[21899]: osdmap e101: 8 total, 8 up, 8 in 2024-04-22T15:19:04.521 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:04 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/1621182036' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/573419087"}]: dispatch 2024-04-22T15:19:04.522 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:04 smithi145 ceph-mon[26606]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:19:04.522 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:04 smithi145 ceph-mon[26606]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/3013561970"}]': finished 2024-04-22T15:19:04.522 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:04 smithi145 ceph-mon[26606]: osdmap e101: 8 total, 8 up, 8 in 2024-04-22T15:19:04.522 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:04 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/1621182036' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/573419087"}]: dispatch 2024-04-22T15:19:04.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:04 smithi172 ceph-mon[22747]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:19:04.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:04 smithi172 ceph-mon[22747]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/3013561970"}]': finished 2024-04-22T15:19:04.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:04 smithi172 ceph-mon[22747]: osdmap e101: 8 total, 8 up, 8 in 2024-04-22T15:19:04.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:04 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/1621182036' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/573419087"}]: dispatch 2024-04-22T15:19:05.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:05 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/1621182036' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/573419087"}]': finished 2024-04-22T15:19:05.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:05 smithi145 ceph-mon[21899]: osdmap e102: 8 total, 8 up, 8 in 2024-04-22T15:19:05.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:05 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/1621182036' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/573419087"}]': finished 2024-04-22T15:19:05.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:05 smithi145 ceph-mon[26606]: osdmap e102: 8 total, 8 up, 8 in 2024-04-22T15:19:05.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/1621182036' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/573419087"}]': finished 2024-04-22T15:19:05.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-mon[22747]: osdmap e102: 8 total, 8 up, 8 in 2024-04-22T15:19:05.718 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:19:05.449+0000 7fd668b72700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:19:05.719 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Creating ceph-iscsi config... 2024-04-22T15:19:05.719 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:19:05.719 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:19:05.719 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:19:05.719 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:19:05.719 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:19:05.719 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Traceback (most recent call last): 2024-04-22T15:19:05.719 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:19:05.719 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: return _run_code(code, main_globals, None, 2024-04-22T15:19:05.719 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:19:05.719 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: exec(code, run_globals) 2024-04-22T15:19:05.719 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:19:05.720 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:19:05.720 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:19:05.720 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:19:05.720 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:19:05.720 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:19:05.720 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:19:05.720 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:19:05.720 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:19:05.720 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Traceback (most recent call last): 2024-04-22T15:19:05.720 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:19:05.720 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:19:05.721 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:19:05.721 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:19:05.721 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:19:05.721 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:19:05.721 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:19:05.721 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: return future.result(timeout) 2024-04-22T15:19:05.721 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:19:05.721 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: return self.__get_result() 2024-04-22T15:19:05.721 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:19:05.721 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: raise self._exception 2024-04-22T15:19:05.721 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:19:05.721 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: use_current_daemon_image=reconfig, 2024-04-22T15:19:05.721 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:19:05.722 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:19:05.722 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:19:05.722 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Creating ceph-iscsi config... 2024-04-22T15:19:05.722 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:19:05.722 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:19:05.722 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:19:05.722 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:19:05.722 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:19:05.722 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Traceback (most recent call last): 2024-04-22T15:19:05.722 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:19:05.722 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: return _run_code(code, main_globals, None, 2024-04-22T15:19:05.723 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:19:05.723 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: exec(code, run_globals) 2024-04-22T15:19:05.723 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:19:05.723 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:19:05.723 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:19:05.723 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:19:05.723 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:19:05.723 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:19:05.723 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:19:05.723 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:19:05.723 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:19:06.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:06 smithi145 ceph-mon[21899]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:19:06.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:06 smithi145 ceph-mon[26606]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:19:06.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:06 smithi172 ceph-mon[22747]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:19:07.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:07 smithi145 ceph-mon[21899]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:19:07.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:19:07.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:19:07.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:19:07.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:19:07.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:19:07.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:19:07.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:19:07.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:19:07.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:19:07.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:19:07.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:19:07.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:19:07.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:19:07.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:19:07.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:19:07.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:19:07.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:19:07.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:19:07.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:19:07.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:19:07.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:19:07.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:19:07.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:19:07.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:19:07.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:19:07.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:19:07.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:19:07.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:19:07.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:19:07.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:19:07.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:19:07.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:19:07.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:19:07.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:19:07.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:19:07.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:19:07.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:19:07.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:19:07.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:19:07.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:19:07.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:19:07.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:19:07.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:19:07.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:19:07.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:19:07.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:19:07.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:19:07.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:19:07.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:19:07.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:19:07.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:19:07.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:19:07.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:19:07.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:19:07.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:19:07.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:19:07.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:19:07.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:19:07.628 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:07 smithi145 ceph-mon[26606]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:19:07.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:19:07.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:19:07.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:19:07.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:19:07.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:19:07.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:19:07.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:19:07.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:19:07.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:19:07.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:19:07.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:19:07.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:19:07.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:19:07.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:19:07.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:19:07.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:19:07.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:19:07.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:19:07.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:19:07.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:19:07.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:19:07.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:19:07.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:19:07.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:19:07.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:19:07.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:19:07.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:19:07.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:19:07.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:19:07.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:19:07.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:19:07.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:19:07.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:19:07.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:19:07.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:19:07.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:19:07.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:19:07.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:19:07.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:19:07.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:19:07.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:19:07.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:19:07.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:19:07.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:19:07.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:19:07.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:19:07.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:19:07.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:19:07.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:19:07.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:19:07.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:19:07.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:19:07.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:19:07.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:19:07.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:19:07.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:19:07.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:19:07.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:19:07.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:07 smithi172 ceph-mon[22747]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:19:07.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:19:07.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:19:07.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:19:07.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:19:07.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:19:07.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:19:07.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:19:07.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:19:07.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:19:07.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:19:07.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:19:07.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:19:07.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:19:07.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:19:07.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:19:07.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:19:07.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:19:07.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:19:07.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:19:07.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:19:07.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:19:07.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:19:07.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:19:07.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:19:07.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:19:07.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:19:07.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:19:07.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:19:07.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:19:07.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:19:07.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:19:07.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:19:07.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:19:07.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:19:07.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:19:07.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:19:07.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:19:07.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:19:07.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:19:07.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:19:07.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:19:07.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:19:07.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:19:07.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:19:07.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:19:07.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:19:07.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:19:07.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:19:07.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:19:07.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:19:07.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:19:07.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:19:07.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:19:07.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:19:07.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:19:07.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:19:07.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:19:07.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:19:08.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:08 smithi145 ceph-mon[21899]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 692 B/s rd, 0 op/s 2024-04-22T15:19:08.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:08 smithi145 ceph-mon[26606]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 692 B/s rd, 0 op/s 2024-04-22T15:19:08.661 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:08 smithi172 ceph-mon[22747]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 692 B/s rd, 0 op/s 2024-04-22T15:19:08.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:19:08 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:19:08.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:19:10.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:10 smithi172 ceph-mon[22747]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 592 B/s rd, 0 op/s 2024-04-22T15:19:10.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:10 smithi145 ceph-mon[21899]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 592 B/s rd, 0 op/s 2024-04-22T15:19:10.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:10 smithi145 ceph-mon[26606]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 592 B/s rd, 0 op/s 2024-04-22T15:19:11.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:11 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:19:11.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:11 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:19:11.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:11 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:19:11.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:11 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:19:11] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.33.4" 2024-04-22T15:19:12.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:12 smithi172 ceph-mon[22747]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:19:12.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:12 smithi145 ceph-mon[21899]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:19:12.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:12 smithi145 ceph-mon[26606]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:19:13.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:19:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:19:13.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:19:14.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:14 smithi172 ceph-mon[22747]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 413 B/s rd, 0 op/s 2024-04-22T15:19:14.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:14 smithi145 ceph-mon[21899]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 413 B/s rd, 0 op/s 2024-04-22T15:19:14.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:14 smithi145 ceph-mon[26606]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 413 B/s rd, 0 op/s 2024-04-22T15:19:16.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:16 smithi172 ceph-mon[22747]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 375 B/s rd, 0 op/s 2024-04-22T15:19:16.545 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:16 smithi145 ceph-mon[26606]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 375 B/s rd, 0 op/s 2024-04-22T15:19:16.546 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:16 smithi145 ceph-mon[21899]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 375 B/s rd, 0 op/s 2024-04-22T15:19:18.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:18 smithi172 ceph-mon[22747]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:19:18.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:18 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2573200541' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T15:19:18.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:18 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2767219994' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2937248060"}]: dispatch 2024-04-22T15:19:18.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:18 smithi172 ceph-mon[22747]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2937248060"}]: dispatch 2024-04-22T15:19:18.524 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:18 smithi145 ceph-mon[21899]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:19:18.525 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:18 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2573200541' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T15:19:18.525 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:18 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2767219994' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2937248060"}]: dispatch 2024-04-22T15:19:18.525 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:18 smithi145 ceph-mon[21899]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2937248060"}]: dispatch 2024-04-22T15:19:18.525 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:18 smithi145 ceph-mon[26606]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:19:18.525 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:18 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2573200541' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T15:19:18.525 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:18 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2767219994' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2937248060"}]: dispatch 2024-04-22T15:19:18.525 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:18 smithi145 ceph-mon[26606]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2937248060"}]: dispatch 2024-04-22T15:19:18.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:19:18 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:19:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:19:19.535 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:19 smithi145 ceph-mon[26606]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2937248060"}]': finished 2024-04-22T15:19:19.535 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:19 smithi145 ceph-mon[26606]: osdmap e103: 8 total, 8 up, 8 in 2024-04-22T15:19:19.535 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:19 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/4017579064' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2469616922"}]: dispatch 2024-04-22T15:19:19.535 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:19 smithi145 ceph-mon[26606]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2469616922"}]: dispatch 2024-04-22T15:19:19.535 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:19 smithi145 ceph-mon[21899]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2937248060"}]': finished 2024-04-22T15:19:19.536 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:19 smithi145 ceph-mon[21899]: osdmap e103: 8 total, 8 up, 8 in 2024-04-22T15:19:19.536 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:19 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/4017579064' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2469616922"}]: dispatch 2024-04-22T15:19:19.536 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:19 smithi145 ceph-mon[21899]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2469616922"}]: dispatch 2024-04-22T15:19:19.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:19 smithi172 ceph-mon[22747]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2937248060"}]': finished 2024-04-22T15:19:19.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:19 smithi172 ceph-mon[22747]: osdmap e103: 8 total, 8 up, 8 in 2024-04-22T15:19:19.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:19 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/4017579064' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2469616922"}]: dispatch 2024-04-22T15:19:19.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:19 smithi172 ceph-mon[22747]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2469616922"}]: dispatch 2024-04-22T15:19:20.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:20 smithi145 ceph-mon[21899]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:19:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:20 smithi145 ceph-mon[21899]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2469616922"}]': finished 2024-04-22T15:19:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:20 smithi145 ceph-mon[21899]: osdmap e104: 8 total, 8 up, 8 in 2024-04-22T15:19:20.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:20 smithi145 ceph-mon[26606]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:19:20.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:20 smithi145 ceph-mon[26606]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2469616922"}]': finished 2024-04-22T15:19:20.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:20 smithi145 ceph-mon[26606]: osdmap e104: 8 total, 8 up, 8 in 2024-04-22T15:19:20.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:20 smithi172 ceph-mon[22747]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 65 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:19:20.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:20 smithi172 ceph-mon[22747]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2469616922"}]': finished 2024-04-22T15:19:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:20 smithi172 ceph-mon[22747]: osdmap e104: 8 total, 8 up, 8 in 2024-04-22T15:19:21.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:21 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:19:21] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.33.4" 2024-04-22T15:19:22.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:22 smithi145 ceph-mon[21899]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-22T15:19:22.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:22 smithi145 ceph-mon[26606]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-22T15:19:22.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:22 smithi172 ceph-mon[22747]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-22T15:19:23.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:19:23 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:19:23.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:19:24.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:24 smithi145 ceph-mon[21899]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-22T15:19:24.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:24 smithi145 ceph-mon[26606]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-22T15:19:24.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:24 smithi172 ceph-mon[22747]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-22T15:19:25.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:25 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:19:25.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:25 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:19:25.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:25 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:19:26.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:26 smithi145 ceph-mon[26606]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-22T15:19:26.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:26 smithi145 ceph-mon[21899]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-22T15:19:26.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:26 smithi172 ceph-mon[22747]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-22T15:19:28.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:28 smithi145 ceph-mon[21899]: from='client.24851 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:19:28.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:28 smithi145 ceph-mon[21899]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:19:28.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:28 smithi145 ceph-mon[26606]: from='client.24851 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:19:28.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:28 smithi145 ceph-mon[26606]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:19:28.661 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:28 smithi172 ceph-mon[22747]: from='client.24851 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:19:28.661 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:28 smithi172 ceph-mon[22747]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:19:28.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:19:28 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:19:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:19:30.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:30 smithi172 ceph-mon[22747]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-04-22T15:19:30.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:30 smithi145 ceph-mon[21899]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-04-22T15:19:30.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:30 smithi145 ceph-mon[26606]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-04-22T15:19:31.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:31 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:19:31] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.33.4" 2024-04-22T15:19:32.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:32 smithi172 ceph-mon[22747]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:19:32.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:32 smithi145 ceph-mon[21899]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:19:32.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:32 smithi145 ceph-mon[26606]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:19:33.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:19:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:19:33.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:19:34.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:34 smithi145 ceph-mon[21899]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:19:34.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:34 smithi145 ceph-mon[26606]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:19:34.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:34 smithi172 ceph-mon[22747]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:19:36.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:36 smithi145 ceph-mon[26606]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:19:36.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:36 smithi145 ceph-mon[21899]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:19:36.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:36 smithi172 ceph-mon[22747]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:19:38.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:38 smithi145 ceph-mon[21899]: from='client.24851 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:19:38.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:38 smithi145 ceph-mon[21899]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:19:38.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:38 smithi145 ceph-mon[26606]: from='client.24851 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:19:38.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:38 smithi145 ceph-mon[26606]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:19:38.661 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:38 smithi172 ceph-mon[22747]: from='client.24851 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:19:38.661 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:38 smithi172 ceph-mon[22747]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:19:38.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:19:38 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:19:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:19:40.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:40 smithi172 ceph-mon[22747]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:19:40.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:40 smithi145 ceph-mon[21899]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:19:40.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:40 smithi145 ceph-mon[26606]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:19:41.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:41 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:19:41.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:41 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:19:41.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:41 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:19:41.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:41 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:19:41] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.33.4" 2024-04-22T15:19:42.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:42 smithi172 ceph-mon[22747]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:19:42.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:42 smithi145 ceph-mon[21899]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:19:42.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:42 smithi145 ceph-mon[26606]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:19:43.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:19:43 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:19:43.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:19:44.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:44 smithi145 ceph-mon[26606]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:19:44.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:44 smithi145 ceph-mon[21899]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:19:44.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:44 smithi172 ceph-mon[22747]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:19:46.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:46 smithi145 ceph-mon[21899]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:19:46.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:46 smithi145 ceph-mon[26606]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:19:46.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:46 smithi172 ceph-mon[22747]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:19:48.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:48 smithi145 ceph-mon[21899]: from='client.24851 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:19:48.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:48 smithi145 ceph-mon[21899]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:19:48.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:48 smithi145 ceph-mon[26606]: from='client.24851 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:19:48.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:48 smithi145 ceph-mon[26606]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:19:48.661 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:48 smithi172 ceph-mon[22747]: from='client.24851 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:19:48.662 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:48 smithi172 ceph-mon[22747]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:19:48.968 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:19:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:19:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:19:50.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:50 smithi172 ceph-mon[22747]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:19:50.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:50 smithi145 ceph-mon[21899]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:19:50.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:50 smithi145 ceph-mon[26606]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:19:51.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:19:51 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:19:51] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.33.4" 2024-04-22T15:19:52.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:52 smithi145 ceph-mon[21899]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:19:52.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:52 smithi145 ceph-mon[26606]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:19:52.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:52 smithi172 ceph-mon[22747]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:19:53.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:19:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:19:53.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:19:54.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:54 smithi145 ceph-mon[21899]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:19:54.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:54 smithi145 ceph-mon[26606]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:19:54.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:54 smithi172 ceph-mon[22747]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:19:56.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:56 smithi145 ceph-mon[21899]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:19:56.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:56 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:19:56.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:56 smithi145 ceph-mon[26606]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:19:56.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:56 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:19:56.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:56 smithi172 ceph-mon[22747]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:19:56.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:56 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:19:58.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:58 smithi145 ceph-mon[21899]: from='client.24851 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:19:58.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:19:58 smithi145 ceph-mon[21899]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:19:58.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:58 smithi145 ceph-mon[26606]: from='client.24851 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:19:58.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:19:58 smithi145 ceph-mon[26606]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:19:58.661 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:58 smithi172 ceph-mon[22747]: from='client.24851 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:19:58.661 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:19:58 smithi172 ceph-mon[22747]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:19:58.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:19:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:19:58.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:20:00.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:00 smithi172 ceph-mon[22747]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:00.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:00 smithi172 ceph-mon[22747]: overall HEALTH_OK 2024-04-22T15:20:00.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:00 smithi145 ceph-mon[21899]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:00.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:00 smithi145 ceph-mon[21899]: overall HEALTH_OK 2024-04-22T15:20:00.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:00 smithi145 ceph-mon[26606]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:00.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:00 smithi145 ceph-mon[26606]: overall HEALTH_OK 2024-04-22T15:20:01.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:20:01] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.33.4" 2024-04-22T15:20:02.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:02 smithi145 ceph-mon[26606]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:20:02.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:02 smithi145 ceph-mon[21899]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:20:02.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:02 smithi172 ceph-mon[22747]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:20:03.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:20:03 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:20:03.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:20:04.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:04 smithi145 ceph-mon[26606]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:04.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:04 smithi145 ceph-mon[21899]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:04.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:04 smithi172 ceph-mon[22747]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:06.270 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:06 smithi172 ceph-mon[22747]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:06.270 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:06 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:20:06.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:06 smithi145 ceph-mon[26606]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:06.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:06 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:20:06.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:06 smithi145 ceph-mon[21899]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:06.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:06 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:20:08.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:08 smithi145 ceph-mon[21899]: from='client.24851 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:20:08.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:08 smithi145 ceph-mon[21899]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:20:08.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:08 smithi145 ceph-mon[26606]: from='client.24851 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:20:08.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:08 smithi145 ceph-mon[26606]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:20:08.486 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:08 smithi172 ceph-mon[22747]: from='client.24851 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:20:08.487 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:08 smithi172 ceph-mon[22747]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:20:08.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:20:08 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:20:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:20:10.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:10 smithi145 ceph-mon[26606]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:10.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:10 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:20:10.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:10 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:20:10.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:10 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:20:10.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:10 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:20:10.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:10 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:20:10.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:10 smithi145 ceph-mon[21899]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:10.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:10 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:20:10.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:10 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:20:10.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:10 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:20:10.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:10 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:20:10.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:10 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:20:10.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:10 smithi172 ceph-mon[22747]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:10.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:10 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:20:10.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:10 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:20:10.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:10 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:20:10.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:10 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:20:10.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:10 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:20:11.324 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:11 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:20:11.324 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:11 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:20:11.324 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:11 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:20:11.324 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:11 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:20:11.324 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:11 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:20:11.324 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:11 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:20:11.325 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:11 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:20:11.325 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:11 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:20:11.325 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:11 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:20:11.325 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:11 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:20:11.325 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:11 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:20:11.325 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:11 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:20:11.687 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:11 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:20:11.687 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:11 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:20:11.687 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:11 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:20:11.687 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:11 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:20:11.687 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:11 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:20:11.687 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:11 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:20:11.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:11 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:20:11] "GET /metrics HTTP/1.1" 200 37897 "" "Prometheus/2.33.4" 2024-04-22T15:20:12.356 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:12 smithi145 ceph-mon[26606]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:20:12.357 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:12 smithi145 ceph-mon[26606]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:20:12.357 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:12 smithi145 ceph-mon[26606]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:20:12.357 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:12 smithi145 ceph-mon[21899]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:20:12.357 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:12 smithi145 ceph-mon[21899]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:20:12.357 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:12 smithi145 ceph-mon[21899]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:20:12.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:12 smithi172 ceph-mon[22747]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:20:12.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:12 smithi172 ceph-mon[22747]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:20:12.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:12 smithi172 ceph-mon[22747]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:20:13.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:20:13.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:20:14.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:14 smithi172 ceph-mon[22747]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:14.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:20:13.998+0000 7fd668b72700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:20:14.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Creating ceph-iscsi config... 2024-04-22T15:20:14.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:20:14.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:20:14.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:20:14.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:20:14.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:20:14.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Traceback (most recent call last): 2024-04-22T15:20:14.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:20:14.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: return _run_code(code, main_globals, None, 2024-04-22T15:20:14.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:20:14.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: exec(code, run_globals) 2024-04-22T15:20:14.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:20:14.470 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:20:14.470 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:20:14.470 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:20:14.470 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:20:14.470 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:20:14.470 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:20:14.470 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:20:14.470 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:20:14.470 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Traceback (most recent call last): 2024-04-22T15:20:14.470 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:20:14.470 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:20:14.470 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:20:14.470 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:20:14.471 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:20:14.471 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:20:14.471 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:20:14.471 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: return future.result(timeout) 2024-04-22T15:20:14.471 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:20:14.471 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: return self.__get_result() 2024-04-22T15:20:14.471 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:20:14.471 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: raise self._exception 2024-04-22T15:20:14.471 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:20:14.471 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: use_current_daemon_image=reconfig, 2024-04-22T15:20:14.471 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:20:14.471 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:20:14.472 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:20:14.472 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Creating ceph-iscsi config... 2024-04-22T15:20:14.472 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:20:14.472 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:20:14.472 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:20:14.472 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:20:14.472 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:20:14.472 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Traceback (most recent call last): 2024-04-22T15:20:14.472 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:20:14.472 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: return _run_code(code, main_globals, None, 2024-04-22T15:20:14.472 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:20:14.473 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: exec(code, run_globals) 2024-04-22T15:20:14.473 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:20:14.473 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:20:14.473 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:20:14.473 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:20:14.473 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:20:14.473 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:20:14.473 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:20:14.473 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:20:14.473 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:20:14.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:14 smithi145 ceph-mon[21899]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:14.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:14 smithi145 ceph-mon[26606]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:15.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:15 smithi145 ceph-mon[21899]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:20:15.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:20:15.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:20:15.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:20:15.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:20:15.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:20:15.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:20:15.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:20:15.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:20:15.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:20:15.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:20:15.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:20:15.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:20:15.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:20:15.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:20:15.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:20:15.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:20:15.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:20:15.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:20:15.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:20:15.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:20:15.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:20:15.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:20:15.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:20:15.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:20:15.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:20:15.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:20:15.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:20:15.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:20:15.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:20:15.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:20:15.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:20:15.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:20:15.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:20:15.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:20:15.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:20:15.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:20:15.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:20:15.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:20:15.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:20:15.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:20:15.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:20:15.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:20:15.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:20:15.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:20:15.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:20:15.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:20:15.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:20:15.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:20:15.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:20:15.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:20:15.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:20:15.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:20:15.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:20:15.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:20:15.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:20:15.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:20:15.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:20:15.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:20:15.628 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:15 smithi145 ceph-mon[26606]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:20:15.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:20:15.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:20:15.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:20:15.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:20:15.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:20:15.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:20:15.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:20:15.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:20:15.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:20:15.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:20:15.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:20:15.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:20:15.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:20:15.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:20:15.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:20:15.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:20:15.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:20:15.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:20:15.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:20:15.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:20:15.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:20:15.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:20:15.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:20:15.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:20:15.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:20:15.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:20:15.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:20:15.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:20:15.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:20:15.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:20:15.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:20:15.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:20:15.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:20:15.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:20:15.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:20:15.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:20:15.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:20:15.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:20:15.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:20:15.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:20:15.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:20:15.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:20:15.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:20:15.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:20:15.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:20:15.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:20:15.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:20:15.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:20:15.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:20:15.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:20:15.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:20:15.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:20:15.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:20:15.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:20:15.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:20:15.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:20:15.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:20:15.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:20:15.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:15 smithi172 ceph-mon[22747]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:20:15.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:20:15.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:20:15.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:20:15.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:20:15.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:20:15.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:20:15.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:20:15.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:20:15.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:20:15.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:20:15.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:20:15.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:20:15.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:20:15.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:20:15.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:20:15.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:20:15.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:20:15.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:20:15.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:20:15.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:20:15.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:20:15.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:20:15.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:20:15.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:20:15.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:20:15.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:20:15.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:20:15.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:20:15.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:20:15.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:20:15.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:20:15.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:20:15.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:20:15.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:20:15.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:20:15.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:20:15.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:20:15.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:20:15.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:20:15.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:20:15.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:20:15.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:20:15.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:20:15.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:20:15.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:20:15.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:20:15.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:20:15.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:20:15.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:20:15.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:20:15.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:20:15.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:20:15.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:20:15.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:20:15.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:20:15.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:20:15.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:20:15.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:20:16.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:16 smithi145 ceph-mon[21899]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:16.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:16 smithi145 ceph-mon[26606]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:16.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:16 smithi172 ceph-mon[22747]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:18.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:18 smithi145 ceph-mon[26606]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-04-22T15:20:18.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:18 smithi145 ceph-mon[21899]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-04-22T15:20:18.661 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:18 smithi172 ceph-mon[22747]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-04-22T15:20:18.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:20:18 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:20:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:20:20.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:20 smithi172 ceph-mon[22747]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T15:20:20.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:20 smithi145 ceph-mon[21899]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T15:20:20.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:20 smithi145 ceph-mon[26606]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T15:20:21.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:21 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:20:21] "GET /metrics HTTP/1.1" 200 37897 "" "Prometheus/2.33.4" 2024-04-22T15:20:22.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:22 smithi145 ceph-mon[21899]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T15:20:22.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:22 smithi145 ceph-mon[26606]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T15:20:22.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:22 smithi172 ceph-mon[22747]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-22T15:20:23.968 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:20:23 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:20:23.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:20:24.389 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:24 smithi145 ceph-mon[21899]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:20:24.390 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:24 smithi145 ceph-mon[26606]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:20:24.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:24 smithi172 ceph-mon[22747]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:20:26.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:26 smithi145 ceph-mon[26606]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:20:26.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:26 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:20:26.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:26 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2162483749' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T15:20:26.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:26 smithi145 ceph-mon[21899]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:20:26.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:26 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:20:26.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:26 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2162483749' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T15:20:26.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:26 smithi172 ceph-mon[22747]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-22T15:20:26.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:26 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:20:26.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:26 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2162483749' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T15:20:28.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:28 smithi145 ceph-mon[21899]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T15:20:28.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:28 smithi145 ceph-mon[26606]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T15:20:28.661 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:28 smithi172 ceph-mon[22747]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-22T15:20:28.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:20:28 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:20:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:20:30.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:30 smithi172 ceph-mon[22747]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-04-22T15:20:30.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:30 smithi145 ceph-mon[26606]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-04-22T15:20:30.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:30 smithi145 ceph-mon[21899]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-04-22T15:20:31.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:31 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:20:31] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.33.4" 2024-04-22T15:20:32.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:32 smithi145 ceph-mon[21899]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:20:32.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:32 smithi145 ceph-mon[26606]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:20:32.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:32 smithi172 ceph-mon[22747]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:20:33.879 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:20:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:20:33.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:20:34.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:34 smithi145 ceph-mon[21899]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:20:34.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:34 smithi145 ceph-mon[26606]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:20:34.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:34 smithi172 ceph-mon[22747]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:20:36.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:36 smithi145 ceph-mon[21899]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:20:36.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:36 smithi145 ceph-mon[26606]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:20:36.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:36 smithi172 ceph-mon[22747]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:20:37.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:37 smithi145 ceph-mon[21899]: from='client.15174 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:20:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:37 smithi145 ceph-mon[26606]: from='client.15174 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:20:37.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:37 smithi172 ceph-mon[22747]: from='client.15174 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:20:38.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:38 smithi145 ceph-mon[21899]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T15:20:38.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:38 smithi145 ceph-mon[26606]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T15:20:38.661 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:38 smithi172 ceph-mon[22747]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T15:20:38.968 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:20:38 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:20:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:20:40.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:40 smithi172 ceph-mon[22747]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:40.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:40 smithi145 ceph-mon[21899]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:40.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:40 smithi145 ceph-mon[26606]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:41.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:41 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:20:41.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:41 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:20:41.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:41 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:20:41.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:41 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:20:41] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.33.4" 2024-04-22T15:20:42.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:42 smithi172 ceph-mon[22747]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:20:42.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:42 smithi145 ceph-mon[21899]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:20:42.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:42 smithi145 ceph-mon[26606]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:20:43.717 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:20:43 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:20:43.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:20:44.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:44 smithi145 ceph-mon[21899]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:44.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:44 smithi145 ceph-mon[26606]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:44.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:44 smithi172 ceph-mon[22747]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:46.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:46 smithi145 ceph-mon[21899]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:46.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:46 smithi145 ceph-mon[26606]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:46.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:46 smithi172 ceph-mon[22747]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:47.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:47 smithi145 ceph-mon[21899]: from='client.15174 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:20:47.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:47 smithi145 ceph-mon[26606]: from='client.15174 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:20:47.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:47 smithi172 ceph-mon[22747]: from='client.15174 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:20:48.524 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:48 smithi145 ceph-mon[21899]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:20:48.524 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:48 smithi145 ceph-mon[26606]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:20:48.661 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:48 smithi172 ceph-mon[22747]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:20:48.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:20:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:20:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:20:50.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:50 smithi172 ceph-mon[22747]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:50.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:50 smithi145 ceph-mon[21899]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:50.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:50 smithi145 ceph-mon[26606]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:51.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:20:51 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:20:51] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.33.4" 2024-04-22T15:20:52.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:52 smithi145 ceph-mon[21899]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:20:52.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:52 smithi145 ceph-mon[26606]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:20:52.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:52 smithi172 ceph-mon[22747]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:20:53.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:20:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:20:53.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:20:54.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:54 smithi145 ceph-mon[21899]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:54.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:54 smithi145 ceph-mon[26606]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:54.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:54 smithi172 ceph-mon[22747]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:55.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:55 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:20:55.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:55 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:20:55.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:55 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:20:56.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:56 smithi145 ceph-mon[21899]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:56.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:56 smithi145 ceph-mon[26606]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:56.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:56 smithi172 ceph-mon[22747]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:20:57.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:57 smithi145 ceph-mon[21899]: from='client.15174 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:20:57.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:57 smithi145 ceph-mon[26606]: from='client.15174 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:20:57.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:57 smithi172 ceph-mon[22747]: from='client.15174 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:20:58.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:20:58 smithi145 ceph-mon[21899]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:20:58.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:20:58 smithi145 ceph-mon[26606]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:20:58.662 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:20:58 smithi172 ceph-mon[22747]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:20:58.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:20:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:20:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:21:00.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:00 smithi172 ceph-mon[22747]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:21:00.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:00 smithi145 ceph-mon[21899]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:21:00.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:00 smithi145 ceph-mon[26606]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:21:01.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:21:01] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.33.4" 2024-04-22T15:21:02.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:02 smithi145 ceph-mon[21899]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:21:02.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:02 smithi145 ceph-mon[26606]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:21:02.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:02 smithi172 ceph-mon[22747]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:21:03.968 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:21:03 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:21:03.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:21:04.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:04 smithi145 ceph-mon[26606]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:21:04.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:04 smithi145 ceph-mon[21899]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:21:04.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:04 smithi172 ceph-mon[22747]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:21:06.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:06 smithi145 ceph-mon[21899]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:21:06.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:06 smithi145 ceph-mon[26606]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:21:06.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:06 smithi172 ceph-mon[22747]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:21:07.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:07 smithi145 ceph-mon[21899]: from='client.15174 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:21:07.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:07 smithi145 ceph-mon[26606]: from='client.15174 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:21:07.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:07 smithi172 ceph-mon[22747]: from='client.15174 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:21:08.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:08 smithi145 ceph-mon[21899]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:21:08.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:08 smithi145 ceph-mon[26606]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:21:08.660 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:08 smithi172 ceph-mon[22747]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:21:08.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:21:08 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:21:08.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:21:10.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:10 smithi172 ceph-mon[22747]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:21:10.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:10 smithi145 ceph-mon[21899]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:21:10.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:10 smithi145 ceph-mon[26606]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:21:11.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:11 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:21:11.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:11 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:21:11.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:11 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:21:11.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:11 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:21:11] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.33.4" 2024-04-22T15:21:12.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:12 smithi172 ceph-mon[22747]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:21:12.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:12 smithi145 ceph-mon[21899]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:21:12.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:12 smithi145 ceph-mon[26606]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:21:13.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:21:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:21:13.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:21:14.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:14 smithi145 ceph-mon[21899]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:21:14.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:14 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:21:14.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:14 smithi145 ceph-mon[26606]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:21:14.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:14 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:21:14.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:14 smithi172 ceph-mon[22747]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:21:14.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:14 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:21:16.513 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:16 smithi172 ceph-mon[22747]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:21:16.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:16 smithi145 ceph-mon[21899]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:21:16.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:16 smithi145 ceph-mon[26606]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:21:17.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:17 smithi145 ceph-mon[21899]: from='client.15174 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:21:17.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:17 smithi145 ceph-mon[26606]: from='client.15174 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:21:17.684 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:17 smithi172 ceph-mon[22747]: from='client.15174 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:21:18.524 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:18 smithi145 ceph-mon[26606]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:21:18.524 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:18 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:18.524 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:18 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:18.524 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:18 smithi145 ceph-mon[21899]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:21:18.524 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:18 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:18.524 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:18 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:18.662 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:18 smithi172 ceph-mon[22747]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:21:18.662 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:18 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:18.662 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:18 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:18.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:21:18 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:21:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:21:19.592 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:19 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:21:19.593 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:19 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:21:19.593 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:19 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:19.593 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:19 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:21:19.593 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:19 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:21:19.593 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:19 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:21:19.593 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:19 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:21:19.593 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:19 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:21:19.593 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:19 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:19.594 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:19 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:21:19.594 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:19 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:21:19.594 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:19 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:21:19.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:19 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:21:19.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:19 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:21:19.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:19 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:19.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:19 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:21:19.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:19 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:21:19.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:19 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:21:20.582 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:20 smithi145 ceph-mon[26606]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:21:20.583 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:20 smithi145 ceph-mon[26606]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:21:20.583 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:20 smithi145 ceph-mon[26606]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:21:20.583 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:20 smithi145 ceph-mon[21899]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:21:20.583 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:20 smithi145 ceph-mon[21899]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:21:20.583 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:20 smithi145 ceph-mon[21899]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:21:20.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:20 smithi172 ceph-mon[22747]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:21:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:20 smithi172 ceph-mon[22747]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:21:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:20 smithi172 ceph-mon[22747]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:21:21.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:21 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:21:21] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.33.4" 2024-04-22T15:21:22.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:22 smithi145 ceph-mon[21899]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:21:22.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:22 smithi145 ceph-mon[26606]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:21:22.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-mon[22747]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:21:22.718 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:21:22.344+0000 7fd668b72700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:21:22.718 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Creating ceph-iscsi config... 2024-04-22T15:21:22.718 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:21:22.718 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:21:22.718 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:21:22.718 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:21:22.718 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:21:22.718 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Traceback (most recent call last): 2024-04-22T15:21:22.718 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:21:22.719 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: return _run_code(code, main_globals, None, 2024-04-22T15:21:22.719 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:21:22.719 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: exec(code, run_globals) 2024-04-22T15:21:22.719 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:21:22.719 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:21:22.719 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:21:22.719 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:21:22.719 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:21:22.719 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:21:22.719 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:21:22.720 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:21:22.720 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:21:22.720 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Traceback (most recent call last): 2024-04-22T15:21:22.720 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:21:22.720 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:21:22.720 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:21:22.720 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:21:22.720 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:21:22.721 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:21:22.721 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:21:22.721 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: return future.result(timeout) 2024-04-22T15:21:22.721 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:21:22.721 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: return self.__get_result() 2024-04-22T15:21:22.721 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:21:22.721 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: raise self._exception 2024-04-22T15:21:22.721 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:21:22.721 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: use_current_daemon_image=reconfig, 2024-04-22T15:21:22.721 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:21:22.721 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:21:22.721 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:21:22.721 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Creating ceph-iscsi config... 2024-04-22T15:21:22.722 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:21:22.722 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:21:22.722 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:21:22.722 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:21:22.722 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:21:22.722 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Traceback (most recent call last): 2024-04-22T15:21:22.722 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:21:22.722 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: return _run_code(code, main_globals, None, 2024-04-22T15:21:22.722 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:21:22.722 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: exec(code, run_globals) 2024-04-22T15:21:22.722 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:21:22.722 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:21:22.722 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:21:22.723 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:21:22.723 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:21:22.723 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:21:22.723 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:21:22.723 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:21:22.723 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:21:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:23 smithi145 ceph-mon[21899]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:21:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:21:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:21:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:21:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:21:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:21:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:21:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:21:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:21:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:21:23.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:21:23.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:21:23.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:21:23.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:21:23.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:21:23.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:21:23.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:21:23.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:21:23.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:21:23.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:21:23.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:21:23.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:21:23.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:21:23.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:21:23.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:21:23.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:21:23.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:21:23.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:21:23.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:21:23.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:21:23.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:21:23.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:21:23.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:21:23.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:21:23.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:21:23.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:21:23.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:21:23.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:21:23.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:21:23.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:21:23.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:21:23.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:21:23.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:21:23.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:21:23.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:21:23.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:21:23.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:21:23.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:21:23.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:21:23.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:21:23.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:21:23.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:21:23.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:21:23.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:21:23.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:21:23.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:21:23.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:21:23.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:21:23.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:21:23.628 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:23 smithi145 ceph-mon[26606]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:21:23.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:21:23.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:21:23.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:21:23.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:21:23.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:21:23.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:21:23.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:21:23.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:21:23.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:21:23.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:21:23.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:21:23.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:21:23.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:21:23.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:21:23.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:21:23.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:21:23.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:21:23.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:21:23.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:21:23.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:21:23.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:21:23.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:21:23.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:21:23.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:21:23.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:21:23.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:21:23.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:21:23.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:21:23.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:21:23.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:21:23.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:21:23.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:21:23.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:21:23.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:21:23.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:21:23.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:21:23.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:21:23.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:21:23.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:21:23.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:21:23.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:21:23.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:21:23.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:21:23.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:21:23.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:21:23.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:21:23.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:21:23.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:21:23.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:21:23.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:21:23.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:21:23.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:21:23.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:21:23.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:21:23.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:21:23.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:21:23.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:21:23.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:21:23.718 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:21:23 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:21:23.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:21:23.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:23 smithi172 ceph-mon[22747]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:21:23.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:21:23.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:21:23.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:21:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:21:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:21:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:21:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:21:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:21:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:21:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:21:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:21:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:21:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:21:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:21:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:21:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:21:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:21:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:21:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:21:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:21:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:21:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:21:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:21:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:21:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:21:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:21:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:21:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:21:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:21:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:21:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:21:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:21:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:21:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:21:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:21:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:21:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:21:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:21:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:21:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:21:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:21:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:21:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:21:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:21:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:21:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:21:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:21:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:21:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:21:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:21:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:21:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:21:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:21:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:21:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:21:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:21:23.723 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:21:23.723 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:21:24.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:24 smithi145 ceph-mon[21899]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T15:21:24.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:24 smithi145 ceph-mon[26606]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T15:21:24.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:24 smithi172 ceph-mon[22747]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T15:21:25.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:25 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:21:25.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:25 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:21:25.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:25 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:21:26.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:26 smithi145 ceph-mon[21899]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T15:21:26.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:26 smithi145 ceph-mon[26606]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T15:21:26.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:26 smithi172 ceph-mon[22747]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T15:21:28.166 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-22T15:21:28.475 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:28 smithi145 ceph-mon[21899]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T15:21:28.476 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:28 smithi145 ceph-mon[26606]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T15:21:28.661 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:28 smithi172 ceph-mon[22747]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T15:21:28.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:21:28 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:21:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:21:30.458 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:30 smithi145 ceph-mon[26606]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:21:30.458 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:30 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/126381159' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-22T15:21:30.459 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:30 smithi145 ceph-mon[21899]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:21:30.459 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:30 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/126381159' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-22T15:21:30.459 INFO:teuthology.orchestra.run.smithi145.stdout:Scheduled to redeploy mgr.y on host 'smithi145' 2024-04-22T15:21:30.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:30 smithi172 ceph-mon[22747]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:21:30.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:30 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/126381159' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-22T15:21:31.091 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch ps --refresh' 2024-04-22T15:21:31.665 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:31 smithi145 ceph-mon[26606]: from='client.15195 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:21:31.665 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:31 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:31.665 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:31 smithi145 ceph-mon[26606]: Schedule redeploy daemon mgr.y 2024-04-22T15:21:31.665 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:31 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:31.665 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:31 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:31.665 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:31 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:21:31.665 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:31 smithi145 ceph-mon[26606]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:21:31.665 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:31 smithi145 ceph-mon[21899]: from='client.15195 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:21:31.665 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:31 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:31.666 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:31 smithi145 ceph-mon[21899]: Schedule redeploy daemon mgr.y 2024-04-22T15:21:31.666 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:31 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:31.666 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:31 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:31.666 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:31 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:21:31.666 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:31 smithi145 ceph-mon[21899]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:21:31.717 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:31 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:21:31] "GET /metrics HTTP/1.1" 200 37906 "" "Prometheus/2.33.4" 2024-04-22T15:21:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:31 smithi172 ceph-mon[22747]: from='client.15195 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:21:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:31 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:31 smithi172 ceph-mon[22747]: Schedule redeploy daemon mgr.y 2024-04-22T15:21:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:31 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:31 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:31 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:21:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:31 smithi172 ceph-mon[22747]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:21:33.378 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:21:33.379 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (2m) 84s ago 10m 14.5M - 0.23.0 ba2b418f427c 2b762f668fa7 2024-04-22T15:21:33.379 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (2m) 15s ago 10m 42.5M - 8.3.5 dad864ee21e9 20a9f6cd1859 2024-04-22T15:21:33.379 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 running (2m) 84s ago 9m 46.3M - 3.5 e1d6a67b021e 1c9e7613cece 2024-04-22T15:21:33.379 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443,9283 running (5m) 15s ago 15m 512M - 19.0.0-3223-g5883d494 3d983c26e278 1920e9785cbb 2024-04-22T15:21:33.379 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:9283 running (16m) 84s ago 16m 411M - 17.2.0 e1d6a67b021e 95f50f021d87 2024-04-22T15:21:33.379 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (16m) 84s ago 16m 56.3M 2048M 17.2.0 e1d6a67b021e 28106a7bf16e 2024-04-22T15:21:33.379 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (15m) 15s ago 15m 49.0M 2048M 17.2.0 e1d6a67b021e a2fc3297953a 2024-04-22T15:21:33.379 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (15m) 84s ago 15m 45.0M 2048M 17.2.0 e1d6a67b021e b71d9b9f8509 2024-04-22T15:21:33.379 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (2m) 84s ago 11m 15.5M - 1.3.1 1dbe0e931976 7a7a9854f247 2024-04-22T15:21:33.379 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (2m) 15s ago 11m 17.8M - 1.3.1 1dbe0e931976 e5b0a512c835 2024-04-22T15:21:33.379 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (14m) 84s ago 14m 54.0M 2992M 17.2.0 e1d6a67b021e ea9cdc639916 2024-04-22T15:21:33.379 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (14m) 84s ago 14m 52.0M 2992M 17.2.0 e1d6a67b021e 3b9b37a8781f 2024-04-22T15:21:33.379 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (14m) 84s ago 14m 48.2M 2992M 17.2.0 e1d6a67b021e d9502dded8f8 2024-04-22T15:21:33.380 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (13m) 84s ago 13m 51.1M 2992M 17.2.0 e1d6a67b021e 8c72f30e54fc 2024-04-22T15:21:33.380 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (13m) 15s ago 13m 53.9M 3248M 17.2.0 e1d6a67b021e 80001f97ed35 2024-04-22T15:21:33.380 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (13m) 15s ago 13m 51.7M 3248M 17.2.0 e1d6a67b021e ec3ab1335b5c 2024-04-22T15:21:33.380 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (12m) 15s ago 12m 51.1M 3248M 17.2.0 e1d6a67b021e 45d475edf1ca 2024-04-22T15:21:33.380 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (12m) 15s ago 12m 50.7M 3248M 17.2.0 e1d6a67b021e cc9e1d9e7916 2024-04-22T15:21:33.380 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (2m) 15s ago 11m 72.9M - 2.33.4 514e6a882f6e b46c16fe4d43 2024-04-22T15:21:33.380 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (9m) 84s ago 9m 74.1M - 17.2.0 e1d6a67b021e c90a636c947e 2024-04-22T15:21:33.380 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (9m) 15s ago 9m 71.7M - 17.2.0 e1d6a67b021e 20ac46e9bfda 2024-04-22T15:21:33.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:21:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:21:33.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:21:34.397 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:34 smithi145 ceph-mon[26606]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:21:34.397 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:34 smithi145 ceph-mon[26606]: from='client.25120 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:21:34.397 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:34 smithi145 ceph-mon[21899]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:21:34.397 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:34 smithi145 ceph-mon[21899]: from='client.25120 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:21:34.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:34 smithi172 ceph-mon[22747]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:21:34.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:34 smithi172 ceph-mon[22747]: from='client.25120 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:21:34.898 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'sleep 180' 2024-04-22T15:21:35.693 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:35 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:35.693 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:35 smithi172 ceph-mon[22747]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:21:35.722 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:35 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:35.722 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:35 smithi145 ceph-mon[26606]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:21:35.723 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:35 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:35.723 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:35 smithi145 ceph-mon[21899]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:21:38.595 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:38 smithi145 ceph-mon[26606]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:21:38.595 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:38 smithi145 ceph-mon[21899]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:21:38.662 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:38 smithi172 ceph-mon[22747]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:21:38.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:21:38 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:21:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:21:39.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:39 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/919254611' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T15:21:39.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:39 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:39.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:39 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:39 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/919254611' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T15:21:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:39 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:39 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:39.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:39 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/919254611' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T15:21:39.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:39 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:39.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:39 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:40.516 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:40 smithi145 ceph-mon[26606]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:21:40.516 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:40 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:21:40.516 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:40 smithi145 ceph-mon[21899]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:21:40.517 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:40 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:21:40.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:40 smithi172 ceph-mon[22747]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:21:40.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:40 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:21:41.813 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:41 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:41.813 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:41 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:41.813 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:41 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:21:41.813 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:41 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:21:41.813 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:41 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:41.813 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:41 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:21:41.813 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:41 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:21:41.813 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:41 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:21:41.813 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:41 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:21:41.814 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:41 smithi145 ceph-mon[21899]: Deploying daemon mgr.y on smithi145 2024-04-22T15:21:41.814 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:41 smithi145 ceph-mon[21899]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 40 KiB/s rd, 0 B/s wr, 66 op/s 2024-04-22T15:21:41.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:41 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:41.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:41 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:41.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:41 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:21:41.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:41 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:21:41.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:41 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:41.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:41 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:21:41.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:41 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:21:41.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:41 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:21:41.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:41 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:21:41.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:41 smithi145 ceph-mon[26606]: Deploying daemon mgr.y on smithi145 2024-04-22T15:21:41.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:41 smithi145 ceph-mon[26606]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 40 KiB/s rd, 0 B/s wr, 66 op/s 2024-04-22T15:21:41.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:41 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:21:41] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.33.4" 2024-04-22T15:21:41.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:41 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:41.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:41 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:41.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:41 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:21:41.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:41 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:21:41.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:41 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:41.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:41 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:21:41.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:41 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:21:41.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:41 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:21:41.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:41 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:21:41.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:41 smithi172 ceph-mon[22747]: Deploying daemon mgr.y on smithi145 2024-04-22T15:21:41.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:41 smithi172 ceph-mon[22747]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 40 KiB/s rd, 0 B/s wr, 66 op/s 2024-04-22T15:21:43.614 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:42 smithi145 systemd[1]: Stopping Ceph mgr.y for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:21:43.614 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:43 smithi145 podman[60745]: 2024-04-22 15:21:43.174586307 +0000 UTC m=+0.163437196 container died 95f50f021d87a9a0c572d17999f705fee933d17dd3d2d43d7e610ea7f4c79dbe (image=quay.io/ceph/ceph:v17.2.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., GIT_CLEAN=True, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , version=8, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, com.redhat.component=centos-stream-container, io.openshift.expose-services=) 2024-04-22T15:21:43.872 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:43 smithi145 podman[60745]: 2024-04-22 15:21:43.718699437 +0000 UTC m=+0.707550316 container cleanup 95f50f021d87a9a0c572d17999f705fee933d17dd3d2d43d7e610ea7f4c79dbe (image=quay.io/ceph/ceph:v17.2.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y, release=754, version=8, ceph=True, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc.) 2024-04-22T15:21:43.872 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:43 smithi145 bash[60745]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y 2024-04-22T15:21:43.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:21:43 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:21:43.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:21:44.155 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:43 smithi145 podman[60760]: 2024-04-22 15:21:43.890381564 +0000 UTC m=+0.719880055 container remove 95f50f021d87a9a0c572d17999f705fee933d17dd3d2d43d7e610ea7f4c79dbe (image=quay.io/ceph/ceph:v17.2.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, vendor=Red Hat, Inc., vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, ceph=True, com.redhat.component=centos-stream-container, io.openshift.expose-services=, name=centos-stream, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, version=8, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, architecture=x86_64, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-22T15:21:44.156 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:43 smithi145 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-04-22T15:21:44.156 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:44 smithi145 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-04-22T15:21:44.156 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:44 smithi145 systemd[1]: Stopped Ceph mgr.y for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:21:44.156 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:44 smithi145 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mgr.y.service: Consumed 1min 1.094s CPU time. 2024-04-22T15:21:44.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:44 smithi145 ceph-mon[21899]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 40 KiB/s rd, 0 B/s wr, 66 op/s 2024-04-22T15:21:44.558 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:44 smithi145 systemd[1]: Starting Ceph mgr.y for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:21:44.558 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:44 smithi145 ceph-mon[26606]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 40 KiB/s rd, 0 B/s wr, 66 op/s 2024-04-22T15:21:44.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:44 smithi172 ceph-mon[22747]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 40 KiB/s rd, 0 B/s wr, 66 op/s 2024-04-22T15:21:44.817 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:44 smithi145 podman[60890]: 2024-04-22 15:21:44.458047823 +0000 UTC m=+0.018419515 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:21:44.818 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:44 smithi145 podman[60890]: 2024-04-22 15:21:44.583313452 +0000 UTC m=+0.143685183 container create b2de97566cf478b98326f7375fb502753c3be42e46892d4d1d3d363935886d49 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T15:21:44.818 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:44 smithi145 podman[60890]: 2024-04-22 15:21:44.815546137 +0000 UTC m=+0.375917829 container init b2de97566cf478b98326f7375fb502753c3be42e46892d4d1d3d363935886d49 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-04-22T15:21:45.122 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:44 smithi145 podman[60890]: 2024-04-22 15:21:44.818404544 +0000 UTC m=+0.378776232 container start b2de97566cf478b98326f7375fb502753c3be42e46892d4d1d3d363935886d49 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-22T15:21:45.122 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:44 smithi145 ceph-mgr[60909]: -- 172.21.15.145:0/2574759957 <== mon.1 v2:172.21.15.145:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x557396ec25a0 con 0x557396e96800 2024-04-22T15:21:45.122 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:44 smithi145 bash[60890]: b2de97566cf478b98326f7375fb502753c3be42e46892d4d1d3d363935886d49 2024-04-22T15:21:45.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:44 smithi145 systemd[1]: Started Ceph mgr.y for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:21:45.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:44 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:21:44.971+0000 7f6662994200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T15:21:45.605 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:21:45.165+0000 7f6662994200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T15:21:45.605 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:21:45.272+0000 7f6662994200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T15:21:45.605 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:21:45.602+0000 7f6662994200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T15:21:45.872 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:21:45.689+0000 7f6662994200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T15:21:46.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:45 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:46.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:45 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:46.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:45 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:46.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:45 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:46.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:45 smithi172 ceph-mon[22747]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:21:46.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:45 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:21:46.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:45 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:21:46.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:45 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:21:46.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:45 smithi172 ceph-mon[22747]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:21:46.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:45 smithi172 ceph-mon[22747]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 40 KiB/s rd, 0 B/s wr, 66 op/s 2024-04-22T15:21:46.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:45 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:46.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:45 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:46.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:45 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:46.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:45 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:46.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:45 smithi145 ceph-mon[21899]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:21:46.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:45 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:21:46.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:45 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:21:46.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:45 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:21:46.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:45 smithi145 ceph-mon[21899]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:21:46.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:45 smithi145 ceph-mon[21899]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 40 KiB/s rd, 0 B/s wr, 66 op/s 2024-04-22T15:21:46.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:45 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:46.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:45 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:46.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:45 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:46.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:45 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:46.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:45 smithi145 ceph-mon[26606]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:21:46.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:45 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:21:46.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:45 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:21:46.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:45 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:21:46.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:45 smithi145 ceph-mon[26606]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:21:46.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:45 smithi145 ceph-mon[26606]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 78 MiB used, 715 GiB / 715 GiB avail; 40 KiB/s rd, 0 B/s wr, 66 op/s 2024-04-22T15:21:47.122 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:21:46.932+0000 7f6662994200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T15:21:47.390 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:21:47.131+0000 7f6662994200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T15:21:47.390 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:21:47.309+0000 7f6662994200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T15:21:47.641 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:21:47.612+0000 7f6662994200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T15:21:47.921 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:21:47.708+0000 7f6662994200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T15:21:48.174 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:21:47.919+0000 7f6662994200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T15:21:48.174 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:21:48.116+0000 7f6662994200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T15:21:48.373 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-mon[22747]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-22T15:21:48.373 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:21:48.366+0000 7fd668b72700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:21:48.373 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Creating ceph-iscsi config... 2024-04-22T15:21:48.373 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:21:48.373 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:21:48.373 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:21:48.373 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:21:48.373 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:21:48.374 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Traceback (most recent call last): 2024-04-22T15:21:48.374 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:21:48.374 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: return _run_code(code, main_globals, None, 2024-04-22T15:21:48.374 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:21:48.374 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: exec(code, run_globals) 2024-04-22T15:21:48.374 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:21:48.374 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:21:48.374 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:21:48.374 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:21:48.374 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:21:48.374 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:21:48.374 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:21:48.375 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:21:48.375 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:21:48.375 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Traceback (most recent call last): 2024-04-22T15:21:48.375 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:21:48.375 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:21:48.375 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:21:48.375 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:21:48.376 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:21:48.376 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:21:48.376 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:21:48.376 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: return future.result(timeout) 2024-04-22T15:21:48.376 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:21:48.376 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: return self.__get_result() 2024-04-22T15:21:48.376 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:21:48.376 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: raise self._exception 2024-04-22T15:21:48.376 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:21:48.376 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: use_current_daemon_image=reconfig, 2024-04-22T15:21:48.376 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:21:48.376 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:21:48.376 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:21:48.376 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Creating ceph-iscsi config... 2024-04-22T15:21:48.376 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:21:48.377 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:21:48.377 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:21:48.377 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:21:48.377 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:21:48.377 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: Traceback (most recent call last): 2024-04-22T15:21:48.377 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:21:48.377 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: return _run_code(code, main_globals, None, 2024-04-22T15:21:48.377 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:21:48.377 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: exec(code, run_globals) 2024-04-22T15:21:48.377 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:21:48.377 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:21:48.377 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:21:48.377 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:21:48.377 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:21:48.377 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:21:48.378 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:21:48.378 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:21:48.378 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:21:48.513 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:48 smithi145 ceph-mon[26606]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-22T15:21:48.513 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:48 smithi145 ceph-mon[21899]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-22T15:21:48.872 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:21:48.759+0000 7f6662994200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T15:21:48.968 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:21:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:21:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:21:49.433 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:49 smithi145 ceph-mon[26606]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:21:49.433 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:21:49.433 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:21:49.434 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:21:49.434 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:21:49.434 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:21:49.434 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:21:49.434 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:21:49.434 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:21:49.434 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:21:49.434 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:21:49.434 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:21:49.434 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:21:49.434 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:21:49.434 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:21:49.434 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:21:49.434 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:21:49.435 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:21:49.435 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:21:49.435 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:21:49.435 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:21:49.435 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:21:49.435 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:21:49.435 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:21:49.435 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:21:49.435 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:21:49.435 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:21:49.435 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:21:49.435 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:21:49.435 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:21:49.435 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:21:49.435 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:21:49.435 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:21:49.436 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:21:49.436 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:21:49.436 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:21:49.436 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:21:49.436 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:21:49.436 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:21:49.437 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:21:49.437 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:21:49.437 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:21:49.437 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:21:49.437 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:21:49.437 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:21:49.437 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:21:49.437 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:21:49.437 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:21:49.437 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:21:49.437 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:21:49.437 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:21:49.438 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:21:49.438 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:21:49.438 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:21:49.438 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:49 smithi145 ceph-mon[21899]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:21:49.438 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:21:49.438 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:21:49.438 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:21:49.438 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:21:49.438 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:21:49.438 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:21:49.439 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:21:49.439 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:21:49.439 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:21:49.439 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:21:49.439 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:21:49.439 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:21:49.439 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:21:49.439 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:21:49.439 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:21:49.439 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:21:49.439 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:21:49.439 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:21:49.439 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:21:49.439 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:21:49.439 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:21:49.439 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:21:49.440 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:21:49.440 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:21:49.440 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:21:49.440 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:21:49.440 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:21:49.440 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:21:49.440 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:21:49.440 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:21:49.440 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:21:49.440 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:21:49.440 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:21:49.440 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:21:49.440 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:21:49.440 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:21:49.441 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:21:49.441 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:21:49.441 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:21:49.441 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:21:49.441 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:21:49.441 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:21:49.441 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:21:49.441 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:21:49.441 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:21:49.441 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:21:49.441 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:21:49.441 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:21:49.441 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:21:49.441 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:21:49.441 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:21:49.441 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:21:49.442 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:21:49.442 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:21:49.442 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:21:49.442 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:21:49.442 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:21:49.442 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:21:49.442 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:49 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:21:49.442 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:21:49.443 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:21:49.443 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:21:49.443 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:21:49.443 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:21:49.443 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:49 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:21:49.443 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:49 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:21:49.216+0000 7f6662994200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T15:21:49.443 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:49 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:21:49.321+0000 7f6662994200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T15:21:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:49 smithi172 ceph-mon[22747]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:21:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:21:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:21:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:21:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:21:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:21:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:21:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:21:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:21:49.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:21:49.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:21:49.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:21:49.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:21:49.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:21:49.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:21:49.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:21:49.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:21:49.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:21:49.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:21:49.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:21:49.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:21:49.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:21:49.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:21:49.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:21:49.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:21:49.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:21:49.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:21:49.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:21:49.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:21:49.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:21:49.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:21:49.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:21:49.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:21:49.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:21:49.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:21:49.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:21:49.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:21:49.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:21:49.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:21:49.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:21:49.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:21:49.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:21:49.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:21:49.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:21:49.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:21:49.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:21:49.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:21:49.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:21:49.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:21:49.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:21:49.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:21:49.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:21:49.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:21:49.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:21:49.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:21:49.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:21:49.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:21:49.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:21:49.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:21:49.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:49 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:21:49.847 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:49 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:21:49.534+0000 7f6662994200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-22T15:21:50.105 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:50 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:21:50.102+0000 7f6662994200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T15:21:50.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:50 smithi145 ceph-mon[26606]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-22T15:21:50.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:50 smithi145 ceph-mon[21899]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-22T15:21:50.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:50 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:21:50.199+0000 7f6662994200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T15:21:50.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:50 smithi172 ceph-mon[22747]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-22T15:21:50.769 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:50 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:21:50.483+0000 7f6662994200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T15:21:50.770 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:50 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:21:50.654+0000 7f6662994200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T15:21:50.770 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:50 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:21:50.766+0000 7f6662994200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T15:21:51.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:50 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:21:50.885+0000 7f6662994200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T15:21:51.372 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:51 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:21:51.200+0000 7f6662994200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T15:21:51.894 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:51 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:21:51.841+0000 7f6662994200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T15:21:51.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:21:51 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:21:51] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.33.4" 2024-04-22T15:21:52.177 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:52 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:21:52.066+0000 7f6662994200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T15:21:52.178 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:52 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:21:52] ENGINE Bus STARTING 2024-04-22T15:21:52.178 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:52 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CherryPy Checker: 2024-04-22T15:21:52.178 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:52 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: The Application mounted at '' has an empty config. 2024-04-22T15:21:52.178 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:52 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:21:52.178 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:52 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:21:52] ENGINE Serving on http://:::9283 2024-04-22T15:21:52.178 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:21:52 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:21:52] ENGINE Bus STARTED 2024-04-22T15:21:52.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:52 smithi145 ceph-mon[26606]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-22T15:21:52.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:52 smithi145 ceph-mon[26606]: Standby manager daemon y restarted 2024-04-22T15:21:52.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:52 smithi145 ceph-mon[26606]: Standby manager daemon y started 2024-04-22T15:21:52.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:52 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-22T15:21:52.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:52 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:21:52.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:52 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-22T15:21:52.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:52 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:21:52.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:52 smithi145 ceph-mon[21899]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-22T15:21:52.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:52 smithi145 ceph-mon[21899]: Standby manager daemon y restarted 2024-04-22T15:21:52.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:52 smithi145 ceph-mon[21899]: Standby manager daemon y started 2024-04-22T15:21:52.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:52 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-22T15:21:52.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:52 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:21:52.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:52 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-22T15:21:52.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:52 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:21:52.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:52 smithi172 ceph-mon[22747]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-22T15:21:52.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:52 smithi172 ceph-mon[22747]: Standby manager daemon y restarted 2024-04-22T15:21:52.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:52 smithi172 ceph-mon[22747]: Standby manager daemon y started 2024-04-22T15:21:52.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:52 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-22T15:21:52.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:52 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:21:52.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:52 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-22T15:21:52.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:52 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:21:53.412 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:53 smithi145 ceph-mon[26606]: mgrmap e27: x(active, since 3m), standbys: y 2024-04-22T15:21:53.412 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:53 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:53.412 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:53 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:53.412 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:53 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:21:53.412 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:53 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:21:53.412 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:53 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:53.412 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:53 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:21:53.412 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:53 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:21:53.412 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:53 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:21:53.413 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:53 smithi145 ceph-mon[21899]: mgrmap e27: x(active, since 3m), standbys: y 2024-04-22T15:21:53.413 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:53 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:53.413 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:53 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:53.413 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:53 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:21:53.413 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:53 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:21:53.413 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:53 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:53.413 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:53 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:21:53.413 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:53 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:21:53.413 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:53 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:21:53.717 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:21:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:21:53.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:21:53.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:53 smithi172 ceph-mon[22747]: mgrmap e27: x(active, since 3m), standbys: y 2024-04-22T15:21:53.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:53 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:53.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:53 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:53.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:53 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:21:53.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:53 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:21:53.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:53 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:53.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:53 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:21:53.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:53 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:21:53.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:53 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:21:54.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:54 smithi145 ceph-mon[21899]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 57 op/s 2024-04-22T15:21:54.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:54 smithi145 ceph-mon[21899]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:21:54.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:54 smithi145 ceph-mon[21899]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:21:54.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:54 smithi145 ceph-mon[21899]: Health check failed: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] (CEPHADM_FAILED_DAEMON) 2024-04-22T15:21:54.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:54 smithi145 ceph-mon[26606]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 57 op/s 2024-04-22T15:21:54.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:54 smithi145 ceph-mon[26606]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:21:54.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:54 smithi145 ceph-mon[26606]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:21:54.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:54 smithi145 ceph-mon[26606]: Health check failed: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] (CEPHADM_FAILED_DAEMON) 2024-04-22T15:21:54.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:54 smithi172 ceph-mon[22747]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 57 op/s 2024-04-22T15:21:54.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:54 smithi172 ceph-mon[22747]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:21:54.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:54 smithi172 ceph-mon[22747]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:21:54.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:54 smithi172 ceph-mon[22747]: Health check failed: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] (CEPHADM_FAILED_DAEMON) 2024-04-22T15:21:56.496 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:56 smithi145 ceph-mon[26606]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 57 op/s 2024-04-22T15:21:56.497 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:56 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:56.497 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:56 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:21:56.497 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:56 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:56.497 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:56 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:56.497 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:56 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T15:21:56.497 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:56 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T15:21:56.497 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:56 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:21:56.497 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:56 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3048890647' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T15:21:56.497 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:56 smithi145 ceph-mon[21899]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 57 op/s 2024-04-22T15:21:56.497 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:56 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:56.497 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:56 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:21:56.498 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:56 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:56.498 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:56 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:56.498 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:56 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T15:21:56.498 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:56 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T15:21:56.498 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:56 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:21:56.498 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:56 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3048890647' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T15:21:56.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:56 smithi172 ceph-mon[22747]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 57 op/s 2024-04-22T15:21:56.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:56 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:56.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:56 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:21:56.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:56 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:56.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:56 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:21:56.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:56 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T15:21:56.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:56 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T15:21:56.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:56 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:21:56.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:56 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/3048890647' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T15:21:57.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:57 smithi145 ceph-mon[21899]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T15:21:57.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:57 smithi145 ceph-mon[21899]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T15:21:57.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:57 smithi145 ceph-mon[26606]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T15:21:57.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:57 smithi145 ceph-mon[26606]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T15:21:57.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:57 smithi172 ceph-mon[22747]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T15:21:57.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:57 smithi172 ceph-mon[22747]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T15:21:58.581 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:21:58 smithi145 ceph-mon[26606]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 57 op/s 2024-04-22T15:21:58.581 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:21:58 smithi145 ceph-mon[21899]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 57 op/s 2024-04-22T15:21:58.661 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:21:58 smithi172 ceph-mon[22747]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 57 op/s 2024-04-22T15:21:58.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:21:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:21:58.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:22:00.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:00 smithi145 ceph-mon[26606]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 273 B/s rd, 0 op/s 2024-04-22T15:22:00.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:00 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:22:00.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:00 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:22:00.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:00 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:22:00.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:00 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:22:00.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:00 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:22:00.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:00 smithi145 ceph-mon[21899]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 273 B/s rd, 0 op/s 2024-04-22T15:22:00.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:00 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:22:00.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:00 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:22:00.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:00 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:22:00.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:00 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:22:00.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:00 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:22:00.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:00 smithi172 ceph-mon[22747]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 273 B/s rd, 0 op/s 2024-04-22T15:22:00.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:00 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:22:00.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:00 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:22:00.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:00 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:22:00.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:00 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:22:00.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:00 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:22:01.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:01 smithi145 ceph-mon[26606]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 286 B/s rd, 0 op/s 2024-04-22T15:22:01.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:01 smithi145 ceph-mon[26606]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state']) 2024-04-22T15:22:01.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:01 smithi145 ceph-mon[26606]: Cluster is now healthy 2024-04-22T15:22:01.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:01 smithi145 ceph-mon[21899]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 286 B/s rd, 0 op/s 2024-04-22T15:22:01.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:01 smithi145 ceph-mon[21899]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state']) 2024-04-22T15:22:01.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:01 smithi145 ceph-mon[21899]: Cluster is now healthy 2024-04-22T15:22:01.686 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:01 smithi172 ceph-mon[22747]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 286 B/s rd, 0 op/s 2024-04-22T15:22:01.686 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:01 smithi172 ceph-mon[22747]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state']) 2024-04-22T15:22:01.686 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:01 smithi172 ceph-mon[22747]: Cluster is now healthy 2024-04-22T15:22:01.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:22:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:22:01] "GET /metrics HTTP/1.1" 200 37983 "" "Prometheus/2.33.4" 2024-04-22T15:22:03.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:03 smithi145 ceph-mon[26606]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 572 B/s rd, 0 op/s 2024-04-22T15:22:03.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:03 smithi145 ceph-mon[21899]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 572 B/s rd, 0 op/s 2024-04-22T15:22:03.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:03 smithi172 ceph-mon[22747]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 572 B/s rd, 0 op/s 2024-04-22T15:22:03.718 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:22:03 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:22:03.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:22:05.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:05 smithi145 ceph-mon[21899]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 533 B/s rd, 0 op/s 2024-04-22T15:22:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:05 smithi145 ceph-mon[26606]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 533 B/s rd, 0 op/s 2024-04-22T15:22:05.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:05 smithi172 ceph-mon[22747]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 533 B/s rd, 0 op/s 2024-04-22T15:22:07.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:07 smithi145 ceph-mon[21899]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:22:07.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:07 smithi145 ceph-mon[21899]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 533 B/s rd, 0 op/s 2024-04-22T15:22:07.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:07 smithi145 ceph-mon[26606]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:22:07.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:07 smithi145 ceph-mon[26606]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 533 B/s rd, 0 op/s 2024-04-22T15:22:07.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:07 smithi172 ceph-mon[22747]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:22:07.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:07 smithi172 ceph-mon[22747]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 533 B/s rd, 0 op/s 2024-04-22T15:22:08.968 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:22:08 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:22:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:22:09.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:09 smithi145 ceph-mon[21899]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 888 B/s rd, 0 op/s 2024-04-22T15:22:09.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:09 smithi145 ceph-mon[26606]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 888 B/s rd, 0 op/s 2024-04-22T15:22:09.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:09 smithi172 ceph-mon[22747]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 888 B/s rd, 0 op/s 2024-04-22T15:22:11.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:11 smithi145 ceph-mon[21899]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 888 B/s rd, 0 op/s 2024-04-22T15:22:11.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:11 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:22:11.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:11 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:22:11.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:11 smithi145 ceph-mon[26606]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 888 B/s rd, 0 op/s 2024-04-22T15:22:11.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:11 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:22:11.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:11 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:22:11.686 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:11 smithi172 ceph-mon[22747]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 888 B/s rd, 0 op/s 2024-04-22T15:22:11.686 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:11 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:22:11.687 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:11 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:22:11.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:22:11 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:22:11] "GET /metrics HTTP/1.1" 200 37981 "" "Prometheus/2.33.4" 2024-04-22T15:22:13.507 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:13 smithi172 ceph-mon[22747]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:22:13.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:13 smithi145 ceph-mon[21899]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:22:13.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:13 smithi145 ceph-mon[26606]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:22:13.969 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:22:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:22:13.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:22:15.524 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:15 smithi145 ceph-mon[26606]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:15.524 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:15 smithi145 ceph-mon[21899]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:15.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:15 smithi172 ceph-mon[22747]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:17.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:17 smithi145 ceph-mon[21899]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:22:17.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:17 smithi145 ceph-mon[21899]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:17.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:17 smithi145 ceph-mon[26606]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:22:17.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:17 smithi145 ceph-mon[26606]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:17.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:17 smithi172 ceph-mon[22747]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:22:17.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:17 smithi172 ceph-mon[22747]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:18.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:22:18 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:22:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:22:19.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:19 smithi145 ceph-mon[21899]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:22:19.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:19 smithi145 ceph-mon[26606]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:22:19.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:19 smithi172 ceph-mon[22747]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:22:21.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:21 smithi145 ceph-mon[21899]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:21.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:21 smithi145 ceph-mon[26606]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:21.686 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:21 smithi172 ceph-mon[22747]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:21.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:22:21 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:22:21] "GET /metrics HTTP/1.1" 200 37981 "" "Prometheus/2.33.4" 2024-04-22T15:22:23.507 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:23 smithi172 ceph-mon[22747]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:22:23.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:23 smithi145 ceph-mon[21899]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:22:23.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:23 smithi145 ceph-mon[26606]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:22:23.968 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:22:23 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:22:23.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:22:25.533 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:25 smithi145 ceph-mon[26606]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:25.533 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:25 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:22:25.533 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:25 smithi145 ceph-mon[21899]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:25.533 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:25 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:22:25.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:25 smithi172 ceph-mon[22747]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:25.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:25 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:22:27.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:27 smithi145 ceph-mon[26606]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:22:27.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:27 smithi145 ceph-mon[26606]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:27.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:27 smithi145 ceph-mon[21899]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:22:27.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:27 smithi145 ceph-mon[21899]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:27.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:27 smithi172 ceph-mon[22747]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:22:27.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:27 smithi172 ceph-mon[22747]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:28.968 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:22:28 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:22:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:22:29.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:29 smithi145 ceph-mon[26606]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:22:29.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:29 smithi145 ceph-mon[21899]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:22:29.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:29 smithi172 ceph-mon[22747]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:22:31.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:31 smithi145 ceph-mon[21899]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:31.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:31 smithi145 ceph-mon[26606]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:31.686 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:31 smithi172 ceph-mon[22747]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:31.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:22:31 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:22:31] "GET /metrics HTTP/1.1" 200 37981 "" "Prometheus/2.33.4" 2024-04-22T15:22:33.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:33 smithi145 ceph-mon[26606]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:22:33.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:33 smithi145 ceph-mon[21899]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:22:33.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:33 smithi172 ceph-mon[22747]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:22:33.719 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:22:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:22:33.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:22:35.544 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:35 smithi145 ceph-mon[21899]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:35.544 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:35 smithi145 ceph-mon[26606]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:35.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:35 smithi172 ceph-mon[22747]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:37.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:37 smithi145 ceph-mon[26606]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:22:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:37 smithi145 ceph-mon[26606]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:37.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:37 smithi145 ceph-mon[21899]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:22:37.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:37 smithi145 ceph-mon[21899]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:37.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:37 smithi172 ceph-mon[22747]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:22:37.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:37 smithi172 ceph-mon[22747]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:38.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:22:38 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:22:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:22:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:39 smithi145 ceph-mon[21899]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:22:39.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:39 smithi145 ceph-mon[26606]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:22:39.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:39 smithi172 ceph-mon[22747]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:22:40.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:40 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:22:40.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:40 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:22:40.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:40 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:22:41.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:41 smithi145 ceph-mon[21899]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:41.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:41 smithi145 ceph-mon[26606]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:41.687 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:41 smithi172 ceph-mon[22747]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:41.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:22:41 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:22:41] "GET /metrics HTTP/1.1" 200 37980 "" "Prometheus/2.33.4" 2024-04-22T15:22:43.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:43 smithi145 ceph-mon[21899]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:22:43.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:43 smithi145 ceph-mon[26606]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:22:43.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:43 smithi172 ceph-mon[22747]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:22:43.718 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:22:43 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:22:43.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:22:45.548 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:45 smithi145 ceph-mon[21899]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:45.548 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:45 smithi145 ceph-mon[26606]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:45.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:45 smithi172 ceph-mon[22747]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:47.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:47 smithi145 ceph-mon[21899]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:22:47.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:47 smithi145 ceph-mon[21899]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:47.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:47 smithi145 ceph-mon[26606]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:22:47.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:47 smithi145 ceph-mon[26606]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:47 smithi172 ceph-mon[22747]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:22:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:47 smithi172 ceph-mon[22747]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:48.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:22:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:22:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:22:49.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:49 smithi145 ceph-mon[21899]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:22:49.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:49 smithi145 ceph-mon[26606]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:22:49.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:49 smithi172 ceph-mon[22747]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:22:51.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:51 smithi145 ceph-mon[21899]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:51.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:51 smithi145 ceph-mon[26606]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:51.686 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:51 smithi172 ceph-mon[22747]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:51.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:22:51 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:22:51] "GET /metrics HTTP/1.1" 200 37980 "" "Prometheus/2.33.4" 2024-04-22T15:22:53.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:53 smithi145 ceph-mon[21899]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:22:53.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:53 smithi145 ceph-mon[26606]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:22:53.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:53 smithi172 ceph-mon[22747]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:22:53.718 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:22:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:22:53.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:22:55.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:55 smithi145 ceph-mon[21899]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:55.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:55 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:22:55.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:55 smithi145 ceph-mon[26606]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:55.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:55 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:22:55.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:55 smithi172 ceph-mon[22747]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:55.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:55 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:22:57.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:57 smithi145 ceph-mon[21899]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:22:57.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:57 smithi145 ceph-mon[21899]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:57.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:57 smithi145 ceph-mon[26606]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:22:57.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:57 smithi145 ceph-mon[26606]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:57.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:57 smithi172 ceph-mon[22747]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:22:57.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:57 smithi172 ceph-mon[22747]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:22:58.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:22:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:22:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:22:59.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:22:59 smithi145 ceph-mon[26606]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:22:59.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:22:59 smithi145 ceph-mon[21899]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:22:59.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:22:59 smithi172 ceph-mon[22747]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:00.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:00 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:23:00.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:00 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:23:00.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:00 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:23:01.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:01 smithi145 ceph-mon[21899]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:01.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:01 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:23:01.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:01 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:23:01.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:01 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:23:01.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:01 smithi145 ceph-mon[26606]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:01.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:01 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:23:01.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:01 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:23:01.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:01 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:23:01.686 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:01 smithi172 ceph-mon[22747]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:01.686 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:01 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:23:01.687 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:01 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:23:01.687 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:01 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:23:01.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:23:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:23:01] "GET /metrics HTTP/1.1" 200 37984 "" "Prometheus/2.33.4" 2024-04-22T15:23:03.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:03 smithi145 ceph-mon[21899]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:03.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:03 smithi145 ceph-mon[26606]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:03.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:03 smithi172 ceph-mon[22747]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:03.718 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:23:03 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:23:03.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:23:05.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:05 smithi145 ceph-mon[21899]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:05 smithi145 ceph-mon[26606]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:05.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:05 smithi172 ceph-mon[22747]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:07.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:07 smithi145 ceph-mon[26606]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:23:07.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:07 smithi145 ceph-mon[26606]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:07.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:07 smithi145 ceph-mon[21899]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:23:07.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:07 smithi145 ceph-mon[21899]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:07.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:07 smithi172 ceph-mon[22747]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:23:07.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:07 smithi172 ceph-mon[22747]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:08.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:23:08 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:23:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:23:09.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:09 smithi145 ceph-mon[21899]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:09.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:09 smithi145 ceph-mon[26606]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:09.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:09 smithi172 ceph-mon[22747]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:10.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:10 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:23:10.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:10 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:23:10.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:10 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:23:11.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:11 smithi145 ceph-mon[21899]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:11.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:11 smithi145 ceph-mon[26606]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:11.687 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:11 smithi172 ceph-mon[22747]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:11.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:23:11 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:23:11] "GET /metrics HTTP/1.1" 200 37985 "" "Prometheus/2.33.4" 2024-04-22T15:23:13.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:13 smithi145 ceph-mon[21899]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:13.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:13 smithi145 ceph-mon[26606]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:13.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:13 smithi172 ceph-mon[22747]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:13.718 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:23:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:23:13.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:23:15.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:15 smithi145 ceph-mon[21899]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:15.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:15 smithi145 ceph-mon[26606]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:15.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:15 smithi172 ceph-mon[22747]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:17.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:17 smithi145 ceph-mon[26606]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:23:17.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:17 smithi145 ceph-mon[26606]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:17.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:17 smithi145 ceph-mon[21899]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:23:17.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:17 smithi145 ceph-mon[21899]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:17.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:17 smithi172 ceph-mon[22747]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:23:17.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:17 smithi172 ceph-mon[22747]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:18.968 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:23:18 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:23:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:23:19.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:19 smithi145 ceph-mon[21899]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:19.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:19 smithi145 ceph-mon[26606]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:19.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:19 smithi172 ceph-mon[22747]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:21.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:21 smithi145 ceph-mon[21899]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:21.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:21 smithi145 ceph-mon[26606]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:21.686 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:21 smithi172 ceph-mon[22747]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:21.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:23:21 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:23:21] "GET /metrics HTTP/1.1" 200 37985 "" "Prometheus/2.33.4" 2024-04-22T15:23:23.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:23 smithi145 ceph-mon[21899]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:23.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:23 smithi145 ceph-mon[26606]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:23.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:23 smithi172 ceph-mon[22747]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:23.718 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:23:23 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:23:23.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:23:25.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:25 smithi145 ceph-mon[21899]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:25.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:25 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:23:25.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:25 smithi145 ceph-mon[26606]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:25.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:25 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:23:25.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:25 smithi172 ceph-mon[22747]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:25.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:25 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:23:27.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:27 smithi145 ceph-mon[21899]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:23:27.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:27 smithi145 ceph-mon[21899]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:27.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:27 smithi145 ceph-mon[26606]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:23:27.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:27 smithi145 ceph-mon[26606]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:27.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:27 smithi172 ceph-mon[22747]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:23:27.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:27 smithi172 ceph-mon[22747]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:28.968 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:23:28 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:23:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:23:29.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:29 smithi172 ceph-mon[22747]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:29.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:29 smithi145 ceph-mon[21899]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:29.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:29 smithi145 ceph-mon[26606]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:31.686 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:31 smithi172 ceph-mon[22747]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:31.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:31 smithi145 ceph-mon[21899]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:31.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:31 smithi145 ceph-mon[26606]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:31.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:23:31 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:23:31] "GET /metrics HTTP/1.1" 200 37982 "" "Prometheus/2.33.4" 2024-04-22T15:23:33.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:33 smithi172 ceph-mon[22747]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:33.718 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:23:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:23:33.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:23:33.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:33 smithi145 ceph-mon[21899]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:33.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:33 smithi145 ceph-mon[26606]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:35.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:35 smithi172 ceph-mon[22747]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:35.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:35 smithi145 ceph-mon[21899]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:35.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:35 smithi145 ceph-mon[26606]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:37.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:37 smithi172 ceph-mon[22747]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:23:37.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:37 smithi172 ceph-mon[22747]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:37.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:37 smithi145 ceph-mon[21899]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:23:37.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:37 smithi145 ceph-mon[21899]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:37.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:37 smithi145 ceph-mon[26606]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:23:37.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:37 smithi145 ceph-mon[26606]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:38.968 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:23:38 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:23:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"172.21.15.172:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.145\", device_class=\"ssd\", hostname=\"smithi145\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.145\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:23:39.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:39 smithi172 ceph-mon[22747]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:39.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:39 smithi145 ceph-mon[21899]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:39.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:39 smithi145 ceph-mon[26606]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:40.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:40 smithi172 ceph-mon[22747]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:40.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:40 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:23:40.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:40 smithi145 ceph-mon[21899]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:40.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:40 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:23:40.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:40 smithi145 ceph-mon[26606]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:40.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:40 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:23:41.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:23:41 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:23:41] "GET /metrics HTTP/1.1" 200 37982 "" "Prometheus/2.33.4" 2024-04-22T15:23:43.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:43 smithi145 ceph-mon[21899]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:43.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:43 smithi145 ceph-mon[26606]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:43.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:43 smithi172 ceph-mon[22747]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:43.967 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:23:43 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:23:43.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"869afcfa-00b9-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-22T15:23:45.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:45 smithi172 ceph-mon[22747]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:45.611 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:45 smithi145 ceph-mon[21899]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:45.611 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:45 smithi145 ceph-mon[26606]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:47.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:47 smithi172 ceph-mon[22747]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:23:47.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:47 smithi172 ceph-mon[22747]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:47.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:47 smithi145 ceph-mon[21899]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:23:47.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:47 smithi145 ceph-mon[21899]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:47.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:47 smithi145 ceph-mon[26606]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:23:47.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:47 smithi145 ceph-mon[26606]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:49.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:49 smithi172 ceph-mon[22747]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:49.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:49 smithi145 ceph-mon[26606]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:49.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:49 smithi145 ceph-mon[21899]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:51.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:51 smithi172 ceph-mon[22747]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:51.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:51 smithi145 ceph-mon[21899]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:51.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:51 smithi145 ceph-mon[26606]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:51.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:23:51 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:23:51] "GET /metrics HTTP/1.1" 200 37982 "" "Prometheus/2.33.4" 2024-04-22T15:23:53.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:53 smithi172 ceph-mon[22747]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:53.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:53 smithi145 ceph-mon[21899]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:53.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:53 smithi145 ceph-mon[26606]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:55.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:55 smithi172 ceph-mon[22747]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:55.621 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:55 smithi145 ceph-mon[21899]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:55.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:55 smithi145 ceph-mon[26606]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:56.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:56 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:23:56.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:56 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:23:56.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:56 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:23:57.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:57 smithi172 ceph-mon[22747]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:23:57.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:57 smithi172 ceph-mon[22747]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:57.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:57 smithi145 ceph-mon[21899]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:23:57.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:57 smithi145 ceph-mon[21899]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:57.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:57 smithi145 ceph-mon[26606]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:23:57.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:57 smithi145 ceph-mon[26606]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:23:59.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:23:59 smithi145 ceph-mon[21899]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:59.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:23:59 smithi145 ceph-mon[26606]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:23:59.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:23:59 smithi172 ceph-mon[22747]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:24:01.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:01 smithi145 ceph-mon[21899]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:01.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:01 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:24:01.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:01 smithi145 ceph-mon[26606]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:01.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:01 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:24:01.686 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:01 smithi172 ceph-mon[22747]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:01.686 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:01 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:24:01.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:24:01] "GET /metrics HTTP/1.1" 200 37980 "" "Prometheus/2.33.4" 2024-04-22T15:24:02.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:02 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:24:02.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:02 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:24:02.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:02 smithi145 ceph-mon[21899]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:24:02.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:02 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:24:02.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:02 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:24:02.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:02 smithi145 ceph-mon[26606]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:24:02.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:02 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:24:02.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:02 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:24:02.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:02 smithi172 ceph-mon[22747]: from='mgr.24743 ' entity='mgr.x' 2024-04-22T15:24:03.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:03 smithi172 ceph-mon[22747]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:24:03.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:03 smithi145 ceph-mon[21899]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:24:03.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:03 smithi145 ceph-mon[26606]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:24:05.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:05 smithi145 ceph-mon[21899]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:05 smithi145 ceph-mon[26606]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:05.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:05 smithi172 ceph-mon[22747]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:07.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:07 smithi145 ceph-mon[26606]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:24:07.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:07 smithi145 ceph-mon[26606]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:07.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:07 smithi145 ceph-mon[21899]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:24:07.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:07 smithi145 ceph-mon[21899]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:07.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:07 smithi172 ceph-mon[22747]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:24:07.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:07 smithi172 ceph-mon[22747]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:09.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:09 smithi145 ceph-mon[26606]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:24:09.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:09 smithi145 ceph-mon[21899]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:24:09.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:09 smithi172 ceph-mon[22747]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:24:11.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:11 smithi145 ceph-mon[21899]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:11.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:11 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:24:11.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:11 smithi145 ceph-mon[26606]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:11.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:11 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:24:11.686 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:11 smithi172 ceph-mon[22747]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:11.687 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:11 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:24:11.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:11 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:24:11] "GET /metrics HTTP/1.1" 200 37977 "" "Prometheus/2.33.4" 2024-04-22T15:24:13.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:13 smithi145 ceph-mon[26606]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:24:13.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:13 smithi145 ceph-mon[21899]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:24:13.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:13 smithi172 ceph-mon[22747]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:24:15.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:15 smithi145 ceph-mon[26606]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:15.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:15 smithi145 ceph-mon[21899]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:15.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:15 smithi172 ceph-mon[22747]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:17.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:17 smithi145 ceph-mon[21899]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:24:17.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:17 smithi145 ceph-mon[21899]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:17.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:17 smithi145 ceph-mon[26606]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:24:17.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:17 smithi145 ceph-mon[26606]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:17.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:17 smithi172 ceph-mon[22747]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:24:17.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:17 smithi172 ceph-mon[22747]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:19.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:19 smithi145 ceph-mon[21899]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:24:19.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:19 smithi145 ceph-mon[26606]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:24:19.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:19 smithi172 ceph-mon[22747]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:24:21.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:21 smithi145 ceph-mon[21899]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:21.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:21 smithi145 ceph-mon[26606]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:21.690 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:21 smithi172 ceph-mon[22747]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:21.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:21 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:24:21] "GET /metrics HTTP/1.1" 200 37977 "" "Prometheus/2.33.4" 2024-04-22T15:24:23.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:23 smithi145 ceph-mon[26606]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:24:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:23 smithi145 ceph-mon[21899]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:24:23.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:23 smithi172 ceph-mon[22747]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:24:25.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:25 smithi145 ceph-mon[21899]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:25.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:25 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:24:25.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:25 smithi145 ceph-mon[26606]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:25.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:25 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:24:25.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:25 smithi172 ceph-mon[22747]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:25.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:25 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:24:27.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:27 smithi145 ceph-mon[21899]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:24:27.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:27 smithi145 ceph-mon[21899]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:27.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:27 smithi145 ceph-mon[26606]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:24:27.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:27 smithi145 ceph-mon[26606]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:27.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:27 smithi172 ceph-mon[22747]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:24:27.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:27 smithi172 ceph-mon[22747]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:29.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:29 smithi145 ceph-mon[21899]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:24:29.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:29 smithi145 ceph-mon[26606]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:24:29.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:29 smithi172 ceph-mon[22747]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:24:31.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:31 smithi145 ceph-mon[21899]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:31.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:31 smithi145 ceph-mon[26606]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:31.686 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:31 smithi172 ceph-mon[22747]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:31.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:31 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:24:31] "GET /metrics HTTP/1.1" 200 37983 "" "Prometheus/2.33.4" 2024-04-22T15:24:33.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:33 smithi145 ceph-mon[21899]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:24:33.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:33 smithi145 ceph-mon[26606]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:24:33.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:33 smithi172 ceph-mon[22747]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:24:35.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:35 smithi145 ceph-mon[26606]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:35.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:35 smithi145 ceph-mon[21899]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:35.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:35 smithi172 ceph-mon[22747]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:37.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:37 smithi145 ceph-mon[21899]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:24:37.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:37 smithi145 ceph-mon[21899]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:37 smithi145 ceph-mon[26606]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:24:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:37 smithi145 ceph-mon[26606]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:37.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:37 smithi172 ceph-mon[22747]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:24:37.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:37 smithi172 ceph-mon[22747]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:38.808 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch ps' 2024-04-22T15:24:39.371 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:39 smithi145 ceph-mon[26606]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:24:39.371 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:39 smithi145 ceph-mon[21899]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:24:39.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:39 smithi172 ceph-mon[22747]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:24:40.363 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:40 smithi145 ceph-mon[26606]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:24:40.364 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:40 smithi145 ceph-mon[21899]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:24:40.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:40 smithi172 ceph-mon[22747]: from='mgr.24743 172.21.15.172:0/3889153828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:24:40.815 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:24:40.815 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (5m) 2m ago 13m 26.1M - 0.23.0 ba2b418f427c 2b762f668fa7 2024-04-22T15:24:40.815 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (5m) 3m ago 13m 47.7M - 8.3.5 dad864ee21e9 20a9f6cd1859 2024-04-22T15:24:40.815 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 running (2m) 2m ago 12m 45.9M - 3.5 e1d6a67b021e b59d1eae5fb0 2024-04-22T15:24:40.816 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443,9283 running (8m) 3m ago 18m 513M - 19.0.0-3223-g5883d494 3d983c26e278 1920e9785cbb 2024-04-22T15:24:40.816 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (2m) 2m ago 19m 450M - 19.0.0-3223-g5883d494 3d983c26e278 b2de97566cf4 2024-04-22T15:24:40.816 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (20m) 2m ago 20m 61.1M 2048M 17.2.0 e1d6a67b021e 28106a7bf16e 2024-04-22T15:24:40.816 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (18m) 3m ago 18m 49.7M 2048M 17.2.0 e1d6a67b021e a2fc3297953a 2024-04-22T15:24:40.816 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (18m) 2m ago 18m 49.0M 2048M 17.2.0 e1d6a67b021e b71d9b9f8509 2024-04-22T15:24:40.816 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (6m) 2m ago 14m 15.4M - 1.3.1 1dbe0e931976 7a7a9854f247 2024-04-22T15:24:40.816 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (5m) 3m ago 14m 18.6M - 1.3.1 1dbe0e931976 e5b0a512c835 2024-04-22T15:24:40.816 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (18m) 2m ago 18m 55.0M 2992M 17.2.0 e1d6a67b021e ea9cdc639916 2024-04-22T15:24:40.816 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (17m) 2m ago 17m 53.3M 2992M 17.2.0 e1d6a67b021e 3b9b37a8781f 2024-04-22T15:24:40.816 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (17m) 2m ago 17m 50.2M 2992M 17.2.0 e1d6a67b021e d9502dded8f8 2024-04-22T15:24:40.816 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (16m) 2m ago 16m 51.3M 2992M 17.2.0 e1d6a67b021e 8c72f30e54fc 2024-04-22T15:24:40.816 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (16m) 3m ago 16m 54.0M 3248M 17.2.0 e1d6a67b021e 80001f97ed35 2024-04-22T15:24:40.816 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (16m) 3m ago 16m 51.7M 3248M 17.2.0 e1d6a67b021e ec3ab1335b5c 2024-04-22T15:24:40.816 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (15m) 3m ago 15m 51.2M 3248M 17.2.0 e1d6a67b021e 45d475edf1ca 2024-04-22T15:24:40.817 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (15m) 3m ago 15m 50.7M 3248M 17.2.0 e1d6a67b021e cc9e1d9e7916 2024-04-22T15:24:40.817 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (5m) 3m ago 14m 72.9M - 2.33.4 514e6a882f6e b46c16fe4d43 2024-04-22T15:24:40.817 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (13m) 2m ago 13m 74.9M - 17.2.0 e1d6a67b021e c90a636c947e 2024-04-22T15:24:40.817 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (13m) 3m ago 12m 71.8M - 17.2.0 e1d6a67b021e 20ac46e9bfda 2024-04-22T15:24:41.518 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:41 smithi145 ceph-mon[21899]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:41.518 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:41 smithi145 ceph-mon[26606]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:41.545 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph versions' 2024-04-22T15:24:41.686 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:41 smithi172 ceph-mon[22747]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:41.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:41 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:24:41] "GET /metrics HTTP/1.1" 200 37981 "" "Prometheus/2.33.4" 2024-04-22T15:24:42.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:42 smithi145 ceph-mon[21899]: from='client.15261 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:24:42.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:42 smithi145 ceph-mon[26606]: from='client.15261 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:24:42.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:42 smithi172 ceph-mon[22747]: from='client.15261 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:24:43.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:43 smithi145 ceph-mon[26606]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:24:43.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:43 smithi145 ceph-mon[21899]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:24:43.663 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:24:43.664 INFO:teuthology.orchestra.run.smithi145.stdout: "mon": { 2024-04-22T15:24:43.664 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-22T15:24:43.664 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:24:43.664 INFO:teuthology.orchestra.run.smithi145.stdout: "mgr": { 2024-04-22T15:24:43.664 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:24:43.665 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:24:43.665 INFO:teuthology.orchestra.run.smithi145.stdout: "osd": { 2024-04-22T15:24:43.665 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T15:24:43.665 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:24:43.665 INFO:teuthology.orchestra.run.smithi145.stdout: "mds": {}, 2024-04-22T15:24:43.665 INFO:teuthology.orchestra.run.smithi145.stdout: "rgw": { 2024-04-22T15:24:43.666 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:24:43.666 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:24:43.666 INFO:teuthology.orchestra.run.smithi145.stdout: "overall": { 2024-04-22T15:24:43.666 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-22T15:24:43.666 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:24:43.666 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:24:43.667 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:24:43.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:43 smithi172 ceph-mon[22747]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:24:44.273 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph health detail' 2024-04-22T15:24:44.583 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:44 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/1142622649' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:44.583 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:44 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/1142622649' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:44.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:44 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/1142622649' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:24:45.491 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:45 smithi145 ceph-mon[21899]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:45.492 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:45 smithi145 ceph-mon[26606]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:45.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:45 smithi172 ceph-mon[22747]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:46.259 INFO:teuthology.orchestra.run.smithi145.stdout:HEALTH_OK 2024-04-22T15:24:46.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:46 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/1521640462' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:24:46.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:46 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/1521640462' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:24:46.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:46 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/1521640462' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:24:46.983 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph -s' 2024-04-22T15:24:47.621 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:47 smithi145 ceph-mon[26606]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:24:47.621 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:47 smithi145 ceph-mon[26606]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:47.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:47 smithi145 ceph-mon[21899]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:24:47.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:47 smithi145 ceph-mon[21899]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:47.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:47 smithi172 ceph-mon[22747]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:24:47.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:47 smithi172 ceph-mon[22747]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:49.067 INFO:teuthology.orchestra.run.smithi145.stdout: cluster: 2024-04-22T15:24:49.067 INFO:teuthology.orchestra.run.smithi145.stdout: id: 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:24:49.067 INFO:teuthology.orchestra.run.smithi145.stdout: health: HEALTH_OK 2024-04-22T15:24:49.067 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-04-22T15:24:49.067 INFO:teuthology.orchestra.run.smithi145.stdout: services: 2024-04-22T15:24:49.067 INFO:teuthology.orchestra.run.smithi145.stdout: mon: 3 daemons, quorum a,c,b (age 18m) 2024-04-22T15:24:49.067 INFO:teuthology.orchestra.run.smithi145.stdout: mgr: x(active, since 6m), standbys: y 2024-04-22T15:24:49.068 INFO:teuthology.orchestra.run.smithi145.stdout: osd: 8 osds: 8 up (since 15m), 8 in (since 15m) 2024-04-22T15:24:49.068 INFO:teuthology.orchestra.run.smithi145.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-04-22T15:24:49.068 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-04-22T15:24:49.068 INFO:teuthology.orchestra.run.smithi145.stdout: data: 2024-04-22T15:24:49.068 INFO:teuthology.orchestra.run.smithi145.stdout: pools: 6 pools, 161 pgs 2024-04-22T15:24:49.068 INFO:teuthology.orchestra.run.smithi145.stdout: objects: 209 objects, 585 KiB 2024-04-22T15:24:49.068 INFO:teuthology.orchestra.run.smithi145.stdout: usage: 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:24:49.068 INFO:teuthology.orchestra.run.smithi145.stdout: pgs: 161 active+clean 2024-04-22T15:24:49.068 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-04-22T15:24:49.068 INFO:teuthology.orchestra.run.smithi145.stdout: io: 2024-04-22T15:24:49.068 INFO:teuthology.orchestra.run.smithi145.stdout: client: 1.2 KiB/s rd, 1 op/s rd, 0 op/s wr 2024-04-22T15:24:49.068 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-04-22T15:24:49.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:49 smithi145 ceph-mon[26606]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:24:49.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:49 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3325693413' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T15:24:49.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:49 smithi145 ceph-mon[21899]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:24:49.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:49 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3325693413' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T15:24:49.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:49 smithi172 ceph-mon[22747]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:24:49.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:49 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/3325693413' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T15:24:49.756 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph mgr fail' 2024-04-22T15:24:51.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:51 smithi145 ceph-mon[21899]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:51.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:51 smithi145 ceph-mon[26606]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:51.687 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:51 smithi172 ceph-mon[22747]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:24:51.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:51 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:24:51] "GET /metrics HTTP/1.1" 200 37981 "" "Prometheus/2.33.4" 2024-04-22T15:24:52.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:52 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/527634174' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-22T15:24:52.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:52 smithi145 ceph-mon[26606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-22T15:24:52.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:52 smithi145 ceph-mon[26606]: osdmap e105: 8 total, 8 up, 8 in 2024-04-22T15:24:52.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:52 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/527634174' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-22T15:24:52.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:52 smithi145 ceph-mon[21899]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-22T15:24:52.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:52 smithi145 ceph-mon[21899]: osdmap e105: 8 total, 8 up, 8 in 2024-04-22T15:24:52.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:24:52 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:24:52] ENGINE Bus STOPPING 2024-04-22T15:24:52.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:24:52 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:24:52] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-22T15:24:52.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:24:52 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:24:52] ENGINE Bus STOPPED 2024-04-22T15:24:52.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:24:52 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:24:52] ENGINE Bus STARTING 2024-04-22T15:24:52.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:24:52 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:24:52] ENGINE Serving on http://:::9283 2024-04-22T15:24:52.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:24:52 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:24:52] ENGINE Bus STARTED 2024-04-22T15:24:52.700 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:52 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/527634174' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-22T15:24:52.701 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:52 smithi172 ceph-mon[22747]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-22T15:24:52.701 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:52 smithi172 ceph-mon[22747]: osdmap e105: 8 total, 8 up, 8 in 2024-04-22T15:24:52.701 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:52 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:24:52.328+0000 7fd6d65b3700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-22T15:24:52.701 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:52 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ignoring --setuser ceph since I am not root 2024-04-22T15:24:52.701 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:52 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ignoring --setgroup ceph since I am not root 2024-04-22T15:24:52.701 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:52 smithi172 ceph-mgr[42978]: -- 172.21.15.172:0/2074615238 <== mon.2 v2:172.21.15.172:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5615ad85c5a0 con 0x5615ad831000 2024-04-22T15:24:52.701 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:52 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:24:52.509+0000 7fb8e7f7a200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T15:24:52.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:52 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:24:52.697+0000 7fb8e7f7a200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T15:24:52.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:52 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:24:52.807+0000 7fb8e7f7a200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T15:24:53.042 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'sleep 180' 2024-04-22T15:24:53.467 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:24:53.137+0000 7fb8e7f7a200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T15:24:53.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:24:53.235+0000 7fb8e7f7a200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T15:24:53.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:53 smithi172 ceph-mon[22747]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-22T15:24:53.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:53 smithi172 ceph-mon[22747]: mgrmap e28: y(active, starting, since 0.520758s) 2024-04-22T15:24:53.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:53 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:24:53.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:53 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:24:53.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:53 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:24:53.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:53 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T15:24:53.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:53 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:24:53.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:53 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:24:53.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:53 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:24:53.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:53 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:24:53.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:53 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:24:53.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:53 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:24:53.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:53 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:24:53.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:53 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:24:53.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:53 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:24:53.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:53 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:24:53.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:53 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:24:53.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:53 smithi172 ceph-mon[22747]: Manager daemon y is now available 2024-04-22T15:24:53.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:53 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:24:53.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:53 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:24:53.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:53 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:24:53.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:53 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:24:53.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[26606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-22T15:24:53.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[26606]: mgrmap e28: y(active, starting, since 0.520758s) 2024-04-22T15:24:53.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:24:53.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:24:53.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:24:53.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T15:24:53.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:24:53.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:24:53.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:24:53.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:24:53.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:24:53.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:24:53.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:24:53.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:24:53.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:24:53.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:24:53.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:24:53.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[26606]: Manager daemon y is now available 2024-04-22T15:24:53.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:24:53.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:24:53.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:24:53.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:24:53.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[21899]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-22T15:24:53.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[21899]: mgrmap e28: y(active, starting, since 0.520758s) 2024-04-22T15:24:53.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:24:53.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:24:53.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:24:53.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T15:24:53.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:24:53.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:24:53.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:24:53.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:24:53.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:24:53.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:24:53.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:24:53.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:24:53.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:24:53.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:24:53.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:24:53.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[21899]: Manager daemon y is now available 2024-04-22T15:24:53.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:24:53.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:24:53.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:24:53.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:53 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:24:54.445 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:54 smithi172 ceph-mon[22747]: [22/Apr/2024:15:24:52] ENGINE Bus STARTING 2024-04-22T15:24:54.445 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:54 smithi172 ceph-mon[22747]: [22/Apr/2024:15:24:52] ENGINE Serving on http://172.21.15.145:8765 2024-04-22T15:24:54.445 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:54 smithi172 ceph-mon[22747]: [22/Apr/2024:15:24:53] ENGINE Serving on https://172.21.15.145:7150 2024-04-22T15:24:54.445 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:54 smithi172 ceph-mon[22747]: [22/Apr/2024:15:24:53] ENGINE Bus STARTED 2024-04-22T15:24:54.446 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:54 smithi172 ceph-mon[22747]: mgrmap e29: y(active, since 1.52833s) 2024-04-22T15:24:54.446 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:54 smithi172 ceph-mon[22747]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:24:54.450 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:54 smithi145 ceph-mon[21899]: [22/Apr/2024:15:24:52] ENGINE Bus STARTING 2024-04-22T15:24:54.450 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:54 smithi145 ceph-mon[21899]: [22/Apr/2024:15:24:52] ENGINE Serving on http://172.21.15.145:8765 2024-04-22T15:24:54.450 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:54 smithi145 ceph-mon[21899]: [22/Apr/2024:15:24:53] ENGINE Serving on https://172.21.15.145:7150 2024-04-22T15:24:54.450 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:54 smithi145 ceph-mon[21899]: [22/Apr/2024:15:24:53] ENGINE Bus STARTED 2024-04-22T15:24:54.450 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:54 smithi145 ceph-mon[21899]: mgrmap e29: y(active, since 1.52833s) 2024-04-22T15:24:54.450 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:54 smithi145 ceph-mon[21899]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:24:54.451 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:54 smithi145 ceph-mon[26606]: [22/Apr/2024:15:24:52] ENGINE Bus STARTING 2024-04-22T15:24:54.451 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:54 smithi145 ceph-mon[26606]: [22/Apr/2024:15:24:52] ENGINE Serving on http://172.21.15.145:8765 2024-04-22T15:24:54.451 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:54 smithi145 ceph-mon[26606]: [22/Apr/2024:15:24:53] ENGINE Serving on https://172.21.15.145:7150 2024-04-22T15:24:54.451 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:54 smithi145 ceph-mon[26606]: [22/Apr/2024:15:24:53] ENGINE Bus STARTED 2024-04-22T15:24:54.451 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:54 smithi145 ceph-mon[26606]: mgrmap e29: y(active, since 1.52833s) 2024-04-22T15:24:54.451 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:54 smithi145 ceph-mon[26606]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:24:54.717 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:54 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:24:54.507+0000 7fb8e7f7a200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T15:24:54.717 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:54 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:24:54.684+0000 7fb8e7f7a200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T15:24:55.215 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:54 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:24:54.886+0000 7fb8e7f7a200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T15:24:55.216 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:55 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:24:55.212+0000 7fb8e7f7a200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T15:24:55.346 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:55 smithi145 ceph-mon[21899]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:24:55.346 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:55 smithi145 ceph-mon[21899]: mgrmap e30: y(active, since 2s) 2024-04-22T15:24:55.467 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:55 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:24:55.311+0000 7fb8e7f7a200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T15:24:55.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:55 smithi172 ceph-mon[22747]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:24:55.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:55 smithi172 ceph-mon[22747]: mgrmap e30: y(active, since 2s) 2024-04-22T15:24:55.598 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:55 smithi145 ceph-mon[26606]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:24:55.599 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:55 smithi145 ceph-mon[26606]: mgrmap e30: y(active, since 2s) 2024-04-22T15:24:55.851 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:55 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:24:55.522+0000 7fb8e7f7a200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T15:24:55.851 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:55 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:24:55.734+0000 7fb8e7f7a200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T15:24:56.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:56 smithi145 ceph-mon[21899]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:24:56.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:56 smithi145 ceph-mon[26606]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:24:56.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:56 smithi172 ceph-mon[22747]: from='client.25165 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:24:56.718 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:56 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:24:56.373+0000 7fb8e7f7a200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T15:24:57.141 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:56 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:24:56.818+0000 7fb8e7f7a200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T15:24:57.141 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:56 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:24:56.919+0000 7fb8e7f7a200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T15:24:57.141 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:24:57.138+0000 7fb8e7f7a200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-22T15:24:57.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:57 smithi172 ceph-mon[22747]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:24:57.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:57 smithi172 ceph-mon[22747]: mgrmap e31: y(active, since 4s) 2024-04-22T15:24:57.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:57 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:24:57.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:57 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:24:57.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:57 smithi145 ceph-mon[21899]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:24:57.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:57 smithi145 ceph-mon[21899]: mgrmap e31: y(active, since 4s) 2024-04-22T15:24:57.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:57 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:24:57.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:57 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:24:57.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:57 smithi145 ceph-mon[26606]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:24:57.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:57 smithi145 ceph-mon[26606]: mgrmap e31: y(active, since 4s) 2024-04-22T15:24:57.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:57 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:24:57.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:57 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:24:57.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:24:57.695+0000 7fb8e7f7a200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T15:24:57.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:24:57.792+0000 7fb8e7f7a200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T15:24:58.312 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:24:58.058+0000 7fb8e7f7a200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T15:24:58.313 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:24:58.205+0000 7fb8e7f7a200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T15:24:58.313 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:24:58.309+0000 7fb8e7f7a200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T15:24:58.717 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:24:58.428+0000 7fb8e7f7a200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T15:24:59.217 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:24:58.739+0000 7fb8e7f7a200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T15:24:59.588 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:59 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:24:59.359+0000 7fb8e7f7a200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T15:24:59.588 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:59 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:24:59.589 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:59 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:24:59.589 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:59 smithi172 ceph-mon[22747]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:24:59.589 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:59 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:24:59.589 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:24:59 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:24:59.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:59 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:24:59.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:59 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:24:59.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:59 smithi145 ceph-mon[26606]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:24:59.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:59 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:24:59.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:24:59 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:24:59.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:59 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:24:59.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:59 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:24:59.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:59 smithi145 ceph-mon[21899]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:24:59.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:59 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:24:59.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:24:59 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:24:59.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:59 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:24:59.585+0000 7fb8e7f7a200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T15:24:59.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:59 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:24:59] ENGINE Bus STARTING 2024-04-22T15:24:59.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:59 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: CherryPy Checker: 2024-04-22T15:24:59.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:59 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: The Application mounted at '' has an empty config. 2024-04-22T15:24:59.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:59 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:24:59.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:59 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:24:59] ENGINE Serving on http://:::9283 2024-04-22T15:24:59.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:24:59 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:24:59] ENGINE Bus STARTED 2024-04-22T15:25:00.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:00 smithi145 ceph-mon[26606]: Standby manager daemon x started 2024-04-22T15:25:00.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:00 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T15:25:00.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:00 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:25:00.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:00 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T15:25:00.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:00 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:25:00.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:00 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:00.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:00 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:00.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:00 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:25:00.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:00 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:25:00.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:00 smithi145 ceph-mon[26606]: Updating smithi145:/etc/ceph/ceph.conf 2024-04-22T15:25:00.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:00 smithi145 ceph-mon[26606]: Updating smithi172:/etc/ceph/ceph.conf 2024-04-22T15:25:00.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:00 smithi145 ceph-mon[21899]: Standby manager daemon x started 2024-04-22T15:25:00.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:00 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T15:25:00.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:00 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:25:00.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:00 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T15:25:00.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:00 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:25:00.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:00 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:00.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:00 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:00.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:00 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:25:00.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:00 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:25:00.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:00 smithi145 ceph-mon[21899]: Updating smithi145:/etc/ceph/ceph.conf 2024-04-22T15:25:00.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:00 smithi145 ceph-mon[21899]: Updating smithi172:/etc/ceph/ceph.conf 2024-04-22T15:25:00.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:00 smithi172 ceph-mon[22747]: Standby manager daemon x started 2024-04-22T15:25:00.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:00 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T15:25:00.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:00 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:25:00.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:00 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T15:25:00.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:00 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:25:00.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:00 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:00.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:00 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:00.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:00 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:25:00.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:00 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:25:00.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:00 smithi172 ceph-mon[22747]: Updating smithi145:/etc/ceph/ceph.conf 2024-04-22T15:25:00.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:00 smithi172 ceph-mon[22747]: Updating smithi172:/etc/ceph/ceph.conf 2024-04-22T15:25:01.604 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:01 smithi145 ceph-mon[26606]: Updating smithi145:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:25:01.604 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:01 smithi145 ceph-mon[26606]: Updating smithi172:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:25:01.604 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:01 smithi145 ceph-mon[26606]: mgrmap e32: y(active, since 8s), standbys: x 2024-04-22T15:25:01.604 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:01 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T15:25:01.604 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:01 smithi145 ceph-mon[26606]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-22T15:25:01.604 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:01 smithi145 ceph-mon[26606]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:25:01.604 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:01 smithi145 ceph-mon[26606]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:25:01.604 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:01 smithi145 ceph-mon[26606]: Updating smithi145:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:25:01.604 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:01 smithi145 ceph-mon[26606]: Updating smithi172:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:25:01.604 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:01 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:01.605 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:01 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:01.605 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:01 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:01.605 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:01 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:01.605 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:01 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:01.605 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:01 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:25:01.605 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:01 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:25:01.605 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:01 smithi145 ceph-mon[21899]: Updating smithi145:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:25:01.605 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:01 smithi145 ceph-mon[21899]: Updating smithi172:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:25:01.605 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:01 smithi145 ceph-mon[21899]: mgrmap e32: y(active, since 8s), standbys: x 2024-04-22T15:25:01.605 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:01 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T15:25:01.605 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:01 smithi145 ceph-mon[21899]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-22T15:25:01.605 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:01 smithi145 ceph-mon[21899]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:25:01.605 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:01 smithi145 ceph-mon[21899]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:25:01.606 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:01 smithi145 ceph-mon[21899]: Updating smithi145:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:25:01.606 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:01 smithi145 ceph-mon[21899]: Updating smithi172:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:25:01.606 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:01 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:01.606 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:01 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:01.606 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:01 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:01.606 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:01 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:01.606 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:01 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:01.606 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:01 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:25:01.606 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:01 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:25:01.687 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:01 smithi172 ceph-mon[22747]: Updating smithi145:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:25:01.687 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:01 smithi172 ceph-mon[22747]: Updating smithi172:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:25:01.687 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:01 smithi172 ceph-mon[22747]: mgrmap e32: y(active, since 8s), standbys: x 2024-04-22T15:25:01.688 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:01 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T15:25:01.688 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:01 smithi172 ceph-mon[22747]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-22T15:25:01.688 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:01 smithi172 ceph-mon[22747]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:25:01.688 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:01 smithi172 ceph-mon[22747]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:25:01.688 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:01 smithi172 ceph-mon[22747]: Updating smithi145:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:25:01.688 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:01 smithi172 ceph-mon[22747]: Updating smithi172:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:25:01.688 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:01 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:01.688 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:01 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:01.688 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:01 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:01.688 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:01 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:01.688 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:01 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:01.688 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:01 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:25:01.688 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:01 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:25:01.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:25:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:25:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T15:25:02.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:02 smithi145 ceph-mon[21899]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:25:02.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:02 smithi145 ceph-mon[21899]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:25:02.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:02 smithi145 ceph-mon[26606]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:25:02.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:02 smithi145 ceph-mon[26606]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:25:02.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:02 smithi172 ceph-mon[22747]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:25:02.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:02 smithi172 ceph-mon[22747]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:25:03.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:03 smithi172 ceph-mon[22747]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-22T15:25:03.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:03 smithi145 ceph-mon[21899]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-22T15:25:03.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:03 smithi145 ceph-mon[26606]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-22T15:25:04.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:25:04.600+0000 7f65e857e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:25:04.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:25:04.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:25:04.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:25:04.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:25:04.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:25:04.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:25:04.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:25:04.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:25:04.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:25:04.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:25:04.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:25:04.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:25:04.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:25:04.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:25:04.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:25:04.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:25:04.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:25:04.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:25:04.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:25:04.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:25:04.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:25:04.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:25:04.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:25:04.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:25:04.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:25:04.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:25:04.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:25:04.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:25:04.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:25:04.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:25:04.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:25:04.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:25:04.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:25:04.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:25:04.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:25:04.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:25:04.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:25:04.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:25:04.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:25:04.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:25:04.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:25:04.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:25:04.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:25:04.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:25:04.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:25:04.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:25:04.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:25:04.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:25:04.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:25:04.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:25:04.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:25:04.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:25:04.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:25:04.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:25:04.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:25:04.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:25:04.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:25:04.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:04 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:25:05.427 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:05 smithi172 ceph-mon[22747]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T15:25:05.427 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:05 smithi172 ceph-mon[22747]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:25:05.427 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:25:05.427 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:25:05.427 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:25:05.427 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:25:05.428 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:25:05.428 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:25:05.428 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:25:05.428 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:25:05.428 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:25:05.428 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:25:05.428 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:25:05.428 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:25:05.428 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:25:05.428 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:25:05.428 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:25:05.428 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:25:05.428 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:25:05.429 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:25:05.429 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:25:05.429 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:25:05.429 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:25:05.429 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:25:05.429 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:25:05.429 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:25:05.429 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:25:05.429 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:25:05.429 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:25:05.429 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:25:05.429 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:25:05.429 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:25:05.429 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:25:05.429 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:25:05.430 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:25:05.430 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:25:05.430 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:25:05.430 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:25:05.430 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:25:05.430 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:25:05.430 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:25:05.430 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:25:05.430 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:25:05.430 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:25:05.430 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:25:05.430 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:25:05.430 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:25:05.430 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:25:05.431 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:25:05.431 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:25:05.431 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:25:05.431 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:25:05.431 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:25:05.431 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:25:05.431 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:25:05.431 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:25:05.431 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:25:05.431 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:25:05.431 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:25:05.431 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:25:05.431 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:05 smithi172 ceph-mon[22747]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T15:25:05.431 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:05 smithi172 ceph-mon[22747]: Reconfiguring daemon prometheus.a on smithi172 2024-04-22T15:25:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:05 smithi145 ceph-mon[26606]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T15:25:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:05 smithi145 ceph-mon[26606]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:25:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:25:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:25:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:25:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:25:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:25:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:25:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:25:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:25:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:25:05.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:25:05.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:25:05.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:25:05.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:25:05.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:25:05.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:25:05.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:25:05.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:25:05.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:25:05.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:25:05.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:25:05.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:25:05.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:25:05.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:25:05.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:25:05.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:25:05.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:25:05.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:25:05.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:25:05.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:25:05.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:25:05.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:25:05.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:25:05.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:25:05.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:25:05.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:25:05.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:25:05.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:25:05.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:25:05.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:25:05.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:25:05.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:25:05.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:25:05.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:25:05.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:25:05.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:25:05.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:25:05.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:25:05.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:25:05.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:25:05.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:25:05.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:25:05.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:25:05.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:25:05.627 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:05 smithi145 ceph-mon[21899]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T15:25:05.627 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:05 smithi145 ceph-mon[21899]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:25:05.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:25:05.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:25:05.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:25:05.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:25:05.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:25:05.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:25:05.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:25:05.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:25:05.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:25:05.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:25:05.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:25:05.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:25:05.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:25:05.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:25:05.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:25:05.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:25:05.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:25:05.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:25:05.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:25:05.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:25:05.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:25:05.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:25:05.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:25:05.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:25:05.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:25:05.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:25:05.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:25:05.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:25:05.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:25:05.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:25:05.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:25:05.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:25:05.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:25:05.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:25:05.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:25:05.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:25:05.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:25:05.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:25:05.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:25:05.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:25:05.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:25:05.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:25:05.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:25:05.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:25:05.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:25:05.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:25:05.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:25:05.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:25:05.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:25:05.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:25:05.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:25:05.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:25:05.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:25:05.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:25:05.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:25:05.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:25:05.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:25:05.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:25:05.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:05 smithi145 ceph-mon[21899]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T15:25:05.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:05 smithi145 ceph-mon[21899]: Reconfiguring daemon prometheus.a on smithi172 2024-04-22T15:25:05.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:25:05.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:25:05.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:25:05.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:25:05.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:25:05.632 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:05 smithi145 ceph-mon[26606]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T15:25:05.632 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:05 smithi145 ceph-mon[26606]: Reconfiguring daemon prometheus.a on smithi172 2024-04-22T15:25:07.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:07 smithi172 ceph-mon[22747]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T15:25:07.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:07 smithi145 ceph-mon[21899]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T15:25:07.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:07 smithi145 ceph-mon[26606]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T15:25:08.332 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:08 smithi172 systemd[1]: Stopping Ceph prometheus.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:25:08.332 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:08 smithi172 bash[51516]: Error: no container with name or ID "ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-22T15:25:08.640 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:08 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:25:08.641 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:08 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:25:08.329Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-22T15:25:08.641 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:08 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:25:08.329Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-22T15:25:08.641 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:08 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:25:08.329Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-22T15:25:08.641 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:08 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:25:08.329Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-22T15:25:08.641 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:08 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:25:08.329Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-22T15:25:08.641 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:08 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:25:08.329Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-22T15:25:08.641 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:08 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:25:08.329Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-22T15:25:08.641 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:08 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:25:08.329Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-22T15:25:08.641 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:08 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:25:08.329Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-22T15:25:08.641 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:08 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:25:08.330Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-22T15:25:08.641 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:08 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:25:08.330Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-22T15:25:08.641 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:08 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[46152]: ts=2024-04-22T15:25:08.330Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-22T15:25:08.642 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:08 smithi172 podman[51528]: 2024-04-22 15:25:08.339483863 +0000 UTC m=+0.124307423 container died b46c16fe4d433151f713ecc1c3b13325a9b66dd6d9368271578f55acc1477ead (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:25:08.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:08 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:25:08.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:08 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:25:08.949 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:08 smithi172 podman[51528]: 2024-04-22 15:25:08.745409743 +0000 UTC m=+0.530233335 container cleanup b46c16fe4d433151f713ecc1c3b13325a9b66dd6d9368271578f55acc1477ead (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:25:08.949 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:08 smithi172 bash[51528]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a 2024-04-22T15:25:08.949 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:08 smithi172 bash[51557]: Error: no container with name or ID "ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-22T15:25:09.210 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:08 smithi172 podman[51541]: 2024-04-22 15:25:08.946500072 +0000 UTC m=+0.604913248 container remove b46c16fe4d433151f713ecc1c3b13325a9b66dd6d9368271578f55acc1477ead (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:25:09.210 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:08 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-22T15:25:09.210 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:08 smithi172 systemd[1]: Stopped Ceph prometheus.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:25:09.210 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:08 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.999s CPU time. 2024-04-22T15:25:09.210 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:08 smithi172 systemd[1]: Starting Ceph prometheus.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:25:09.210 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:09 smithi172 podman[51626]: 2024-04-22 15:25:09.082047033 +0000 UTC m=+0.019307013 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-22T15:25:09.211 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:09 smithi172 podman[51626]: 2024-04-22 15:25:09.20581177 +0000 UTC m=+0.143071737 container create 6e956501159b424eba9e689e286ddcf241d763c8e30e7207adec753ca2468d4c (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:25:09.461 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:09 smithi172 ceph-mon[22747]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T15:25:09.461 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:09 smithi172 podman[51626]: 2024-04-22 15:25:09.433378906 +0000 UTC m=+0.370638873 container init 6e956501159b424eba9e689e286ddcf241d763c8e30e7207adec753ca2468d4c (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:25:09.461 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:09 smithi172 podman[51626]: 2024-04-22 15:25:09.436048729 +0000 UTC m=+0.373308699 container start 6e956501159b424eba9e689e286ddcf241d763c8e30e7207adec753ca2468d4c (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:25:09.718 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:09 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:25:09.458Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-22T15:25:09.718 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:09 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:25:09.458Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-22T15:25:09.718 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:09 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:25:09.458Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-22T15:25:09.718 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:09 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:25:09.458Z caller=main.go:518 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi172 (none))" 2024-04-22T15:25:09.718 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:09 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:25:09.458Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-22T15:25:09.718 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:09 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:25:09.458Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-22T15:25:09.719 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:09 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:25:09.460Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-22T15:25:09.719 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:09 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:25:09.461Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-22T15:25:09.719 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:09 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:25:09.461Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-22T15:25:09.719 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:09 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:25:09.463Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-22T15:25:09.719 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:09 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:25:09.463Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.54µs 2024-04-22T15:25:09.719 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:09 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:25:09.463Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-22T15:25:09.719 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:09 smithi172 bash[51626]: 6e956501159b424eba9e689e286ddcf241d763c8e30e7207adec753ca2468d4c 2024-04-22T15:25:09.719 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:09 smithi172 systemd[1]: Started Ceph prometheus.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:25:09.719 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:09 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:25:09.653Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-04-22T15:25:09.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:09 smithi145 ceph-mon[21899]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T15:25:09.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:09 smithi145 ceph-mon[26606]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T15:25:10.218 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:09 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:25:09.871Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-04-22T15:25:10.218 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:10 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:25:10.056Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-04-22T15:25:10.218 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:10 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:25:10.057Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-04-22T15:25:10.218 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:10 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:25:10.057Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=29.628µs wal_replay_duration=593.850472ms total_replay_duration=593.892048ms 2024-04-22T15:25:10.218 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:10 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:25:10.062Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-22T15:25:10.218 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:10 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:25:10.062Z caller=main.go:947 level=info msg="TSDB started" 2024-04-22T15:25:10.218 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:10 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:25:10.062Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-22T15:25:10.218 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:10 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:25:10.086Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=24.856123ms db_storage=837ns remote_storage=1.234µs web_handler=503ns query_engine=809ns scrape=472.764µs scrape_sd=132.176µs notify=10.976µs notify_sd=9.481µs rules=23.785554ms 2024-04-22T15:25:10.218 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:25:10 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:25:10.087Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-22T15:25:10.689 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:10 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:10.689 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:10 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:10.689 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:10 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:25:10.689 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:10 smithi172 ceph-mon[22747]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:25:10.689 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:10 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:25:10.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:10 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:10.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:10 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:10.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:10 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:25:10.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:10 smithi145 ceph-mon[21899]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:25:10.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:10 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:25:10.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:10 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:10.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:10 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:10.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:10 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:25:10.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:10 smithi145 ceph-mon[26606]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:25:10.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:10 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:25:11.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:11 smithi145 ceph-mon[21899]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T15:25:11.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:11 smithi145 ceph-mon[26606]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T15:25:11.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:11 smithi172 ceph-mon[22747]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-22T15:25:13.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:13 smithi172 ceph-mon[22747]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:25:13.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:13 smithi145 ceph-mon[21899]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:25:13.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:13 smithi145 ceph-mon[26606]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:25:15.291 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:14 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:15.291 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:14 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:15.291 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:14 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:25:15.291 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:14 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:25:15.291 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:14 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:15.292 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:14 smithi145 ceph-mon[26606]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:25:15.292 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:14 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:25:15.292 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:14 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:25:15.292 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:14 smithi145 ceph-mon[26606]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:25:15.292 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:14 smithi145 ceph-mon[26606]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:25:15.292 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:14 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:15.292 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:14 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:15.292 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:14 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:25:15.292 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:14 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:25:15.292 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:14 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:15.292 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:14 smithi145 ceph-mon[21899]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:25:15.293 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:14 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:25:15.293 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:14 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:25:15.293 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:14 smithi145 ceph-mon[21899]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:25:15.293 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:14 smithi145 ceph-mon[21899]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:25:15.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:14 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:15.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:14 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:15.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:14 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:25:15.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:14 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:25:15.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:14 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:25:15.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:14 smithi172 ceph-mon[22747]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:25:15.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:14 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:25:15.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:14 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:25:15.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:14 smithi172 ceph-mon[22747]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:25:15.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:14 smithi172 ceph-mon[22747]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:25:15.545 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:25:15] "GET /metrics HTTP/1.1" 200 37994 "" "Prometheus/2.33.4" 2024-04-22T15:25:17.588 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-mon[26606]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:25:17.588 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-mon[21899]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:25:17.589 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:25:17.264+0000 7f65e857e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-869afcfa-00b9-11ef-bc93-c7b262605968-iscsi-foo-smithi145-ljpvqb 2024-04-22T15:25:17.589 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: /usr/bin/podman: stderr Error: no such container ceph-869afcfa-00b9-11ef-bc93-c7b262605968-iscsi-foo-smithi145-ljpvqb 2024-04-22T15:25:17.589 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-869afcfa-00b9-11ef-bc93-c7b262605968-iscsi.foo.smithi145.ljpvqb 2024-04-22T15:25:17.589 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: /usr/bin/podman: stderr Error: no such container ceph-869afcfa-00b9-11ef-bc93-c7b262605968-iscsi.foo.smithi145.ljpvqb 2024-04-22T15:25:17.589 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:25:17.589 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:25:17.589 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:25:17.589 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:25:17.589 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:25:17.590 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:25:17.590 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:25:17.590 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:25:17.590 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:25:17.590 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:25:17.590 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:25:17.590 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:25:17.590 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:25:17.590 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:25:17.590 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:25:17.590 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:25:17.590 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:25:17.590 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:25:17.591 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:25:17.591 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:25:17.591 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:25:17.591 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:25:17.591 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:25:17.591 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:25:17.591 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:25:17.591 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:25:17.591 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:25:17.591 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:25:17.591 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:25:17.591 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:25:17.591 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:25:17.591 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:25:17.591 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:25:17.592 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:25:17.592 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:25:17.592 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:25:17.592 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:25:17.592 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:25:17.592 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-869afcfa-00b9-11ef-bc93-c7b262605968-iscsi-foo-smithi145-ljpvqb 2024-04-22T15:25:17.592 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: /usr/bin/podman: stderr Error: no such container ceph-869afcfa-00b9-11ef-bc93-c7b262605968-iscsi-foo-smithi145-ljpvqb 2024-04-22T15:25:17.593 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-869afcfa-00b9-11ef-bc93-c7b262605968-iscsi.foo.smithi145.ljpvqb 2024-04-22T15:25:17.593 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: /usr/bin/podman: stderr Error: no such container ceph-869afcfa-00b9-11ef-bc93-c7b262605968-iscsi.foo.smithi145.ljpvqb 2024-04-22T15:25:17.593 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:25:17.593 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:25:17.593 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:25:17.593 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:25:17.593 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:25:17.593 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:25:17.593 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:25:17.593 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:25:17.593 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:25:17.593 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:25:17.593 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:25:17.594 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:25:17.594 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:25:17.594 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:25:17.594 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:25:17.594 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:25:17.594 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:25:17.594 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:25:17.594 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:25:17.594 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:25:17.594 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:25:17.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:17 smithi172 ceph-mon[22747]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:25:18.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:18 smithi145 ceph-mon[21899]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-869afcfa-00b9-11ef-bc93-c7b262605968-iscsi-foo-smithi145-ljpvqb 2024-04-22T15:25:18.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: /usr/bin/podman: stderr Error: no such container ceph-869afcfa-00b9-11ef-bc93-c7b262605968-iscsi-foo-smithi145-ljpvqb 2024-04-22T15:25:18.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-869afcfa-00b9-11ef-bc93-c7b262605968-iscsi.foo.smithi145.ljpvqb 2024-04-22T15:25:18.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: /usr/bin/podman: stderr Error: no such container ceph-869afcfa-00b9-11ef-bc93-c7b262605968-iscsi.foo.smithi145.ljpvqb 2024-04-22T15:25:18.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:25:18.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:25:18.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:25:18.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:25:18.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:25:18.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:25:18.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:25:18.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:25:18.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:25:18.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:25:18.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:25:18.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:25:18.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:25:18.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:25:18.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:25:18.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:25:18.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:25:18.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:25:18.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:25:18.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:25:18.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:25:18.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:25:18.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:25:18.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:25:18.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:25:18.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:25:18.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:25:18.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:25:18.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:25:18.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:25:18.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:25:18.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:25:18.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:25:18.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:25:18.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:25:18.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:25:18.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:25:18.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:25:18.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-869afcfa-00b9-11ef-bc93-c7b262605968-iscsi-foo-smithi145-ljpvqb 2024-04-22T15:25:18.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: /usr/bin/podman: stderr Error: no such container ceph-869afcfa-00b9-11ef-bc93-c7b262605968-iscsi-foo-smithi145-ljpvqb 2024-04-22T15:25:18.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-869afcfa-00b9-11ef-bc93-c7b262605968-iscsi.foo.smithi145.ljpvqb 2024-04-22T15:25:18.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: /usr/bin/podman: stderr Error: no such container ceph-869afcfa-00b9-11ef-bc93-c7b262605968-iscsi.foo.smithi145.ljpvqb 2024-04-22T15:25:18.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:25:18.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:25:18.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:25:18.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:25:18.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:25:18.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:25:18.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:25:18.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:25:18.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:25:18.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:25:18.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:25:18.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:25:18.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:25:18.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:25:18.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:25:18.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:25:18.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:25:18.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:25:18.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:25:18.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:25:18.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:25:18.628 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:18 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2254741884' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T15:25:18.629 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:18 smithi145 ceph-mon[26606]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-869afcfa-00b9-11ef-bc93-c7b262605968-iscsi-foo-smithi145-ljpvqb 2024-04-22T15:25:18.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: /usr/bin/podman: stderr Error: no such container ceph-869afcfa-00b9-11ef-bc93-c7b262605968-iscsi-foo-smithi145-ljpvqb 2024-04-22T15:25:18.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-869afcfa-00b9-11ef-bc93-c7b262605968-iscsi.foo.smithi145.ljpvqb 2024-04-22T15:25:18.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: /usr/bin/podman: stderr Error: no such container ceph-869afcfa-00b9-11ef-bc93-c7b262605968-iscsi.foo.smithi145.ljpvqb 2024-04-22T15:25:18.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:25:18.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:25:18.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:25:18.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:25:18.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:25:18.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:25:18.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:25:18.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:25:18.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:25:18.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:25:18.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:25:18.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:25:18.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:25:18.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:25:18.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:25:18.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:25:18.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:25:18.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:25:18.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:25:18.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:25:18.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:25:18.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:25:18.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:25:18.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:25:18.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:25:18.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:25:18.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:25:18.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:25:18.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:25:18.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:25:18.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:25:18.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:25:18.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:25:18.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:25:18.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:25:18.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:25:18.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:25:18.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:25:18.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-869afcfa-00b9-11ef-bc93-c7b262605968-iscsi-foo-smithi145-ljpvqb 2024-04-22T15:25:18.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: /usr/bin/podman: stderr Error: no such container ceph-869afcfa-00b9-11ef-bc93-c7b262605968-iscsi-foo-smithi145-ljpvqb 2024-04-22T15:25:18.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-869afcfa-00b9-11ef-bc93-c7b262605968-iscsi.foo.smithi145.ljpvqb 2024-04-22T15:25:18.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: /usr/bin/podman: stderr Error: no such container ceph-869afcfa-00b9-11ef-bc93-c7b262605968-iscsi.foo.smithi145.ljpvqb 2024-04-22T15:25:18.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:25:18.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:25:18.634 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:25:18.634 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:25:18.634 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:25:18.634 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:25:18.634 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:25:18.634 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:25:18.634 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:25:18.634 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:25:18.634 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:25:18.634 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:25:18.634 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:25:18.634 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:25:18.634 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:25:18.634 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:25:18.635 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:25:18.635 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:25:18.635 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:25:18.635 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:25:18.635 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:25:18.635 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:18 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2254741884' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T15:25:18.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:18 smithi172 ceph-mon[22747]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-869afcfa-00b9-11ef-bc93-c7b262605968-iscsi-foo-smithi145-ljpvqb 2024-04-22T15:25:18.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: /usr/bin/podman: stderr Error: no such container ceph-869afcfa-00b9-11ef-bc93-c7b262605968-iscsi-foo-smithi145-ljpvqb 2024-04-22T15:25:18.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-869afcfa-00b9-11ef-bc93-c7b262605968-iscsi.foo.smithi145.ljpvqb 2024-04-22T15:25:18.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: /usr/bin/podman: stderr Error: no such container ceph-869afcfa-00b9-11ef-bc93-c7b262605968-iscsi.foo.smithi145.ljpvqb 2024-04-22T15:25:18.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:25:18.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:25:18.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:25:18.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:25:18.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:25:18.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:25:18.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:25:18.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:25:18.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:25:18.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:25:18.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:25:18.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:25:18.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:25:18.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:25:18.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:25:18.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:25:18.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:25:18.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:25:18.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:25:18.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:25:18.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:25:18.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:25:18.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:25:18.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:25:18.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:25:18.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:25:18.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:25:18.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:25:18.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:25:18.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:25:18.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:25:18.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:25:18.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:25:18.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:25:18.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:25:18.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:25:18.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:25:18.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:25:18.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-869afcfa-00b9-11ef-bc93-c7b262605968-iscsi-foo-smithi145-ljpvqb 2024-04-22T15:25:18.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: /usr/bin/podman: stderr Error: no such container ceph-869afcfa-00b9-11ef-bc93-c7b262605968-iscsi-foo-smithi145-ljpvqb 2024-04-22T15:25:18.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-869afcfa-00b9-11ef-bc93-c7b262605968-iscsi.foo.smithi145.ljpvqb 2024-04-22T15:25:18.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: /usr/bin/podman: stderr Error: no such container ceph-869afcfa-00b9-11ef-bc93-c7b262605968-iscsi.foo.smithi145.ljpvqb 2024-04-22T15:25:18.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:25:18.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:25:18.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:25:18.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:25:18.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:25:18.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:25:18.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:25:18.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:25:18.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:25:18.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:25:18.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:25:18.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:25:18.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:25:18.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:25:18.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:25:18.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:25:18.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:25:18.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:25:18.723 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:25:18.723 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:25:18.723 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:25:18.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:18 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2254741884' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T15:25:19.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:19 smithi172 ceph-mon[22747]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:25:19.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:19 smithi145 ceph-mon[21899]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:25:19.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:19 smithi145 ceph-mon[26606]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:25:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:21 smithi172 ceph-mon[22747]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-04-22T15:25:21.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:21 smithi145 ceph-mon[21899]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-04-22T15:25:21.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:21 smithi145 ceph-mon[26606]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-04-22T15:25:23.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:23 smithi172 ceph-mon[22747]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-04-22T15:25:23.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:23 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:25:23.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:23 smithi145 ceph-mon[21899]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-04-22T15:25:23.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:23 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:25:23.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:23 smithi145 ceph-mon[26606]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-04-22T15:25:23.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:23 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:25:25.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:25 smithi172 ceph-mon[22747]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:25:25.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:25 smithi145 ceph-mon[21899]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:25:25.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:25 smithi145 ceph-mon[26606]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:25:25.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:25:25] "GET /metrics HTTP/1.1" 200 37990 "" "Prometheus/2.33.4" 2024-04-22T15:25:27.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:27 smithi172 ceph-mon[22747]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:25:27.861 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:27 smithi145 ceph-mon[21899]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:25:27.862 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:27 smithi145 ceph-mon[26606]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:25:28.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:28 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:25:28.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:28 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:25:28.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:28 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:25:29.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:29 smithi172 ceph-mon[22747]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:25:29.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:29 smithi145 ceph-mon[26606]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:25:29.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:29 smithi145 ceph-mon[21899]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-22T15:25:31.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:31 smithi172 ceph-mon[22747]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T15:25:31.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:31 smithi145 ceph-mon[21899]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T15:25:31.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:31 smithi145 ceph-mon[26606]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-22T15:25:33.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:33 smithi172 ceph-mon[22747]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:25:33.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:33 smithi145 ceph-mon[21899]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:25:33.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:33 smithi145 ceph-mon[26606]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:25:35.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:35 smithi145 ceph-mon[21899]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:25:35.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:35 smithi145 ceph-mon[26606]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:25:35.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:25:35] "GET /metrics HTTP/1.1" 200 37990 "" "Prometheus/2.33.4" 2024-04-22T15:25:35.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:35 smithi172 ceph-mon[22747]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:25:37.660 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:37 smithi145 ceph-mon[26606]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:25:37.660 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:37 smithi145 ceph-mon[21899]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:25:37.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:37 smithi172 ceph-mon[22747]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:25:38.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:38 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:25:38.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:38 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:25:38.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:38 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:25:38.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:38 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:25:38.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:38 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:25:38.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:38 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:25:39.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:39 smithi172 ceph-mon[22747]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:25:39.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:39 smithi145 ceph-mon[21899]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:25:39.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:39 smithi145 ceph-mon[26606]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:25:41.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:41 smithi172 ceph-mon[22747]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:25:41.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:41 smithi145 ceph-mon[21899]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:25:41.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:41 smithi145 ceph-mon[26606]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:25:43.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:43 smithi172 ceph-mon[22747]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:25:43.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:43 smithi145 ceph-mon[21899]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:25:43.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:43 smithi145 ceph-mon[26606]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:25:45.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:45 smithi145 ceph-mon[21899]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:25:45.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:45 smithi145 ceph-mon[26606]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:25:45.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:25:45] "GET /metrics HTTP/1.1" 200 37990 "" "Prometheus/2.33.4" 2024-04-22T15:25:45.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:45 smithi172 ceph-mon[22747]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:25:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:47 smithi172 ceph-mon[22747]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:25:47.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:47 smithi145 ceph-mon[21899]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:25:47.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:47 smithi145 ceph-mon[26606]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:25:48.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:48 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:25:48.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:48 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:25:48.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:48 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:25:49.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:49 smithi172 ceph-mon[22747]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:25:49.774 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:49 smithi145 ceph-mon[21899]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:25:49.774 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:49 smithi145 ceph-mon[26606]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:25:51.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:51 smithi172 ceph-mon[22747]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:25:51.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:51 smithi145 ceph-mon[21899]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:25:51.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:51 smithi145 ceph-mon[26606]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:25:53.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:53 smithi172 ceph-mon[22747]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:25:53.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:53 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:25:53.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:53 smithi145 ceph-mon[26606]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:25:53.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:53 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:25:53.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:53 smithi145 ceph-mon[21899]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:25:53.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:53 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:25:55.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:55 smithi145 ceph-mon[21899]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:25:55.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:55 smithi145 ceph-mon[26606]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:25:55.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:25:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:25:55] "GET /metrics HTTP/1.1" 200 37984 "" "Prometheus/2.33.4" 2024-04-22T15:25:55.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:55 smithi172 ceph-mon[22747]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:25:57.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:57 smithi172 ceph-mon[22747]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:25:57.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:57 smithi145 ceph-mon[21899]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:25:57.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:57 smithi145 ceph-mon[26606]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:25:59.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:59 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:25:59.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:25:59 smithi172 ceph-mon[22747]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:25:59.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:59 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:25:59.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:25:59 smithi145 ceph-mon[26606]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:25:59.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:59 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:25:59.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:25:59 smithi145 ceph-mon[21899]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:01.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:01 smithi172 ceph-mon[22747]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:01.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:01 smithi145 ceph-mon[26606]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:01.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:01 smithi145 ceph-mon[21899]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:03.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:03 smithi172 ceph-mon[22747]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:03.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:03 smithi145 ceph-mon[21899]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:03.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:03 smithi145 ceph-mon[26606]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:05.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:05 smithi145 ceph-mon[21899]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:05 smithi145 ceph-mon[26606]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:05.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:05 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:26:05] "GET /metrics HTTP/1.1" 200 37984 "" "Prometheus/2.33.4" 2024-04-22T15:26:05.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:05 smithi172 ceph-mon[22747]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:07.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:07 smithi172 ceph-mon[22747]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:07.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:07 smithi145 ceph-mon[21899]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:07.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:07 smithi145 ceph-mon[26606]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:08.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:08 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:26:08.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:08 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:26:08.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:08 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:26:09.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:09 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:26:09.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:09 smithi172 ceph-mon[22747]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:09.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:09 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:26:09.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:09 smithi145 ceph-mon[21899]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:09.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:09 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:26:09.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:09 smithi145 ceph-mon[26606]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:11.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:11 smithi172 ceph-mon[22747]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:11.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:11 smithi145 ceph-mon[26606]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:11.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:11 smithi145 ceph-mon[21899]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:13.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:13 smithi172 ceph-mon[22747]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:13.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:13 smithi145 ceph-mon[21899]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:13.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:13 smithi145 ceph-mon[26606]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:15.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:15 smithi145 ceph-mon[21899]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:15.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:15 smithi145 ceph-mon[26606]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:15.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:26:15] "GET /metrics HTTP/1.1" 200 37987 "" "Prometheus/2.33.4" 2024-04-22T15:26:15.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:15 smithi172 ceph-mon[22747]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:17.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:17 smithi172 ceph-mon[22747]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:17.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:17 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:26:17.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:17 smithi145 ceph-mon[26606]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:17.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:17 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:26:17.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:17 smithi145 ceph-mon[21899]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:17.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:17 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:26:19.096 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:18 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:26:19.096 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:18 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:26:19.096 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:18 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:26:19.096 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:18 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:26:19.096 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:18 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:26:19.096 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:18 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:26:19.096 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:18 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:26:19.097 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:18 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:26:19.097 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:18 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:26:19.097 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:18 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:26:19.097 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:18 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:26:19.097 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:18 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:26:19.097 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:18 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:26:19.097 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:18 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:26:19.097 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:18 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:26:19.097 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:18 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:26:19.097 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:18 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:26:19.097 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:18 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:26:19.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:18 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:26:19.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:18 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:26:19.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:18 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:26:19.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:18 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:26:19.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:18 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:26:19.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:18 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:26:19.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:18 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:26:19.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:18 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:26:19.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:18 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:26:20.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:19 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:26:20.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:19 smithi145 ceph-mon[21899]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:20.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:19 smithi145 ceph-mon[21899]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:26:20.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:19 smithi145 ceph-mon[21899]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:26:20.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:19 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:26:20.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:19 smithi145 ceph-mon[26606]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:20.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:19 smithi145 ceph-mon[26606]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:26:20.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:19 smithi145 ceph-mon[26606]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:26:20.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:19 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:26:20.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:19 smithi172 ceph-mon[22747]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:20.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:19 smithi172 ceph-mon[22747]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:26:20.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:19 smithi172 ceph-mon[22747]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:26:20.470 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:26:20.321+0000 7f65e857e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:26:20.471 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:26:20.471 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:26:20.471 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:26:20.471 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:26:20.471 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:26:20.471 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:26:20.471 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:26:20.471 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:26:20.471 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:26:20.471 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:26:20.471 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:26:20.471 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:26:20.472 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:26:20.472 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:26:20.472 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:26:20.472 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:26:20.472 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:26:20.472 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:26:20.472 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:26:20.472 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:26:20.472 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:26:20.472 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:26:20.472 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:26:20.472 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:26:20.472 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:26:20.472 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:26:20.473 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:26:20.473 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:26:20.473 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:26:20.473 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:26:20.473 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:26:20.473 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:26:20.473 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:26:20.473 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:26:20.473 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:26:20.473 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:26:20.473 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:26:20.473 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:26:20.473 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:26:20.473 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:26:20.474 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:26:20.474 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:26:20.474 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:26:20.474 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:26:20.474 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:26:20.474 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:26:20.474 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:26:20.474 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:26:20.474 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:26:20.475 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:26:20.475 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:26:20.475 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:26:20.475 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:26:20.475 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:26:20.475 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:26:20.475 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:26:20.475 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:26:20.475 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:20 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:26:22.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:21 smithi145 ceph-mon[21899]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:26:22.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:26:22.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:26:22.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:26:22.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:26:22.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:26:22.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:26:22.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:26:22.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:26:22.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:26:22.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:26:22.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:26:22.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:26:22.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:26:22.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:26:22.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:26:22.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:26:22.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:26:22.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:26:22.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:26:22.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:26:22.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:26:22.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:26:22.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:26:22.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:26:22.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:26:22.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:26:22.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:26:22.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:26:22.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:26:22.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:26:22.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:26:22.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:26:22.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:26:22.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:26:22.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:26:22.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:26:22.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:26:22.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:26:22.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:26:22.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:26:22.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:26:22.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:26:22.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:26:22.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:26:22.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:26:22.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:26:22.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:26:22.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:26:22.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:26:22.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:26:22.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:26:22.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:26:22.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:26:22.127 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:21 smithi145 ceph-mon[26606]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:26:22.127 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:26:22.127 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:26:22.127 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:26:22.127 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:26:22.127 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:26:22.127 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:26:22.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:26:22.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:26:22.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:26:22.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:26:22.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:26:22.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:26:22.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:26:22.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:26:22.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:26:22.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:26:22.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:26:22.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:26:22.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:26:22.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:26:22.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:26:22.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:26:22.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:26:22.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:26:22.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:26:22.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:26:22.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:26:22.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:26:22.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:26:22.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:26:22.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:26:22.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:26:22.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:26:22.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:26:22.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:26:22.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:26:22.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:26:22.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:26:22.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:26:22.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:26:22.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:26:22.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:26:22.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:26:22.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:26:22.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:26:22.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:26:22.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:26:22.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:26:22.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:26:22.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:26:22.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:26:22.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:26:22.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:26:22.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:26:22.132 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:26:22.132 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:26:22.132 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:26:22.132 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:26:22.132 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:21 smithi145 ceph-mon[26606]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:22.132 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:26:22.132 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:26:22.132 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:26:22.132 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:26:22.132 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:26:22.132 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:21 smithi145 ceph-mon[21899]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:22.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:21 smithi172 ceph-mon[22747]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:26:22.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:26:22.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:26:22.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:26:22.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:26:22.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:26:22.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:26:22.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:26:22.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:26:22.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:26:22.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:26:22.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:26:22.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:26:22.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:26:22.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:26:22.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:26:22.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:26:22.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:26:22.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:26:22.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:26:22.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:26:22.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:26:22.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:26:22.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:26:22.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:26:22.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:26:22.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:26:22.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:26:22.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:26:22.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:26:22.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:26:22.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:26:22.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:26:22.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:26:22.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:26:22.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:26:22.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:26:22.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:26:22.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:26:22.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:26:22.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:26:22.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:26:22.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:26:22.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:26:22.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:26:22.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:26:22.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:26:22.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:26:22.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:26:22.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:26:22.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:26:22.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:26:22.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:26:22.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:26:22.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:26:22.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:26:22.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:26:22.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:26:22.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:26:22.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:21 smithi172 ceph-mon[22747]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:23.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:22 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:26:23.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:22 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:26:23.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:22 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:26:24.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:23 smithi145 ceph-mon[21899]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:24.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:23 smithi145 ceph-mon[26606]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:24.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:23 smithi172 ceph-mon[22747]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:25.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:25 smithi145 ceph-mon[21899]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:25.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:25 smithi145 ceph-mon[26606]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:25.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:26:25] "GET /metrics HTTP/1.1" 200 37987 "" "Prometheus/2.33.4" 2024-04-22T15:26:25.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:25 smithi172 ceph-mon[22747]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:27.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:27 smithi172 ceph-mon[22747]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:27.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:27 smithi145 ceph-mon[21899]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:27.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:27 smithi145 ceph-mon[26606]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:29.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:29 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:26:29.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:29 smithi172 ceph-mon[22747]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:29.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:29 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:26:29.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:29 smithi145 ceph-mon[21899]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:29.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:29 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:26:29.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:29 smithi145 ceph-mon[26606]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:31.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:31 smithi172 ceph-mon[22747]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:31.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:31 smithi145 ceph-mon[26606]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:31.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:31 smithi145 ceph-mon[21899]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:33.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:33 smithi172 ceph-mon[22747]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:33.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:33 smithi145 ceph-mon[21899]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:33.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:33 smithi145 ceph-mon[26606]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:35.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:35 smithi145 ceph-mon[21899]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:35.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:35 smithi145 ceph-mon[26606]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:35.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:26:35] "GET /metrics HTTP/1.1" 200 37987 "" "Prometheus/2.33.4" 2024-04-22T15:26:35.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:35 smithi172 ceph-mon[22747]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:37.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:37 smithi172 ceph-mon[22747]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:37.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:37 smithi145 ceph-mon[21899]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:37.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:37 smithi145 ceph-mon[26606]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:38.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:38 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:26:38.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:38 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:26:38.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:38 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:26:39.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:39 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:26:39.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:39 smithi172 ceph-mon[22747]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:39.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:39 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:26:39.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:39 smithi145 ceph-mon[21899]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:39.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:39 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:26:39.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:39 smithi145 ceph-mon[26606]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:41.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:41 smithi172 ceph-mon[22747]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:41.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:41 smithi145 ceph-mon[21899]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:41.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:41 smithi145 ceph-mon[26606]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:43.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:43 smithi172 ceph-mon[22747]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:43.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:43 smithi145 ceph-mon[21899]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:43.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:43 smithi145 ceph-mon[26606]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:45.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:45 smithi145 ceph-mon[21899]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:45.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:45 smithi145 ceph-mon[26606]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:45.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:26:45] "GET /metrics HTTP/1.1" 200 37988 "" "Prometheus/2.33.4" 2024-04-22T15:26:45.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:45 smithi172 ceph-mon[22747]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:47 smithi172 ceph-mon[22747]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:47.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:47 smithi145 ceph-mon[21899]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:47.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:47 smithi145 ceph-mon[26606]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:49.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:49 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:26:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:49 smithi172 ceph-mon[22747]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:49.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:49 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:26:49.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:49 smithi145 ceph-mon[21899]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:49.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:49 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:26:49.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:49 smithi145 ceph-mon[26606]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:51.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:51 smithi172 ceph-mon[22747]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:51.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:51 smithi145 ceph-mon[26606]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:51.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:51 smithi145 ceph-mon[21899]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:52.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:52 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:26:52.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:52 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:26:52.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:52 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:26:53.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:53 smithi172 ceph-mon[22747]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:53.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:53 smithi145 ceph-mon[21899]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:53.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:53 smithi145 ceph-mon[26606]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:55.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:55 smithi145 ceph-mon[21899]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:55.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:55 smithi145 ceph-mon[26606]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:55.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:26:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:26:55] "GET /metrics HTTP/1.1" 200 37987 "" "Prometheus/2.33.4" 2024-04-22T15:26:55.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:55 smithi172 ceph-mon[22747]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:57.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:57 smithi172 ceph-mon[22747]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:57.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:57 smithi145 ceph-mon[21899]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:57.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:57 smithi145 ceph-mon[26606]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:26:59.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:59 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:26:59.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:26:59 smithi172 ceph-mon[22747]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:59.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:59 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:26:59.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:26:59 smithi145 ceph-mon[21899]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:26:59.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:59 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:26:59.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:26:59 smithi145 ceph-mon[26606]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:01.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:01 smithi172 ceph-mon[22747]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:01.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:01 smithi145 ceph-mon[21899]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:01.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:01 smithi145 ceph-mon[26606]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:03.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:03 smithi172 ceph-mon[22747]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:03.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:03 smithi145 ceph-mon[26606]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:03.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:03 smithi145 ceph-mon[21899]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:05.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:05 smithi145 ceph-mon[21899]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:05 smithi145 ceph-mon[26606]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:05.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:05 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:27:05] "GET /metrics HTTP/1.1" 200 37987 "" "Prometheus/2.33.4" 2024-04-22T15:27:05.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:05 smithi172 ceph-mon[22747]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:07.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:07 smithi172 ceph-mon[22747]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:07.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:07 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:27:07.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:07 smithi145 ceph-mon[26606]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:07.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:07 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:27:07.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:07 smithi145 ceph-mon[21899]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:07.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:07 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:27:09.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:09 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:27:09.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:09 smithi172 ceph-mon[22747]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:09.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:09 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:27:09.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:09 smithi145 ceph-mon[21899]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:09.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:09 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:27:09.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:09 smithi145 ceph-mon[26606]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:11.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:11 smithi172 ceph-mon[22747]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:11.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:11 smithi145 ceph-mon[21899]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:11.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:11 smithi145 ceph-mon[26606]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:13.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:13 smithi172 ceph-mon[22747]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:13.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:13 smithi145 ceph-mon[21899]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:13.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:13 smithi145 ceph-mon[26606]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:15.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:15 smithi145 ceph-mon[26606]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:15.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:15 smithi145 ceph-mon[21899]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:15.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:27:15] "GET /metrics HTTP/1.1" 200 37988 "" "Prometheus/2.33.4" 2024-04-22T15:27:15.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:15 smithi172 ceph-mon[22747]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:17.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:17 smithi172 ceph-mon[22747]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:17.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:17 smithi145 ceph-mon[21899]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:17.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:17 smithi145 ceph-mon[26606]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:19.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:19 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:27:19.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:19 smithi172 ceph-mon[22747]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:19.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:19 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:27:19.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:19 smithi145 ceph-mon[26606]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:19.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:19 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:27:19.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:19 smithi145 ceph-mon[21899]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:20.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:20 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:27:20.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:20 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:27:20.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:20 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:27:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:21 smithi172 ceph-mon[22747]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:21 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:27:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:21 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:27:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:21 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:27:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:21 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:27:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:21 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:27:21.763 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:21 smithi145 ceph-mon[21899]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:21.764 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:21 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:27:21.764 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:21 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:27:21.764 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:21 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:27:21.764 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:21 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:27:21.764 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:21 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:27:21.764 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:21 smithi145 ceph-mon[26606]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:21.764 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:21 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:27:21.764 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:21 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:27:21.764 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:21 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' 2024-04-22T15:27:21.764 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:21 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:27:21.765 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:21 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:27:22.715 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-mon[26606]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:27:22.716 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-mon[26606]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:27:22.716 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:27:22.716 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-mon[21899]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:27:22.716 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-mon[21899]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:27:22.716 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:27:22.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:22 smithi172 ceph-mon[22747]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:27:22.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:22 smithi172 ceph-mon[22747]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:27:22.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:22 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:27:23.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:27:22.792+0000 7f65e857e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:27:23.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:27:23.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:27:23.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:27:23.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:27:23.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:27:23.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:27:23.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:27:23.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:27:23.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:27:23.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:27:23.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:27:23.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:27:23.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:27:23.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:27:23.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:27:23.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:27:23.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:27:23.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:27:23.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:27:23.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:27:23.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:27:23.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:27:23.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:27:23.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:27:23.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:27:23.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:27:23.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:27:23.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:27:23.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:27:23.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:27:23.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:27:23.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:27:23.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:27:23.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:27:23.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:27:23.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:27:23.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:27:23.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:27:23.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:27:23.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:27:23.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:27:23.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:27:23.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:27:23.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:27:23.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:27:23.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:27:23.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:27:23.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:27:23.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:27:23.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:27:23.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:27:23.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:27:23.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:27:23.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:27:23.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:27:23.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:27:23.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:27:23.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:27:23.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:23 smithi172 ceph-mon[22747]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:23.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:23 smithi172 ceph-mon[22747]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:27:23.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:27:23.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:27:23.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:27:23.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:27:23.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:27:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:27:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:27:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:27:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:27:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:27:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:27:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:27:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:27:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:27:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:27:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:27:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:27:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:27:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:27:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:27:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:27:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:27:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:27:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:27:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:27:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:27:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:27:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:27:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:27:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:27:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:27:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:27:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:27:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:27:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:27:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:27:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:27:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:27:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:27:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:27:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:27:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:27:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:27:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:27:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:27:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:27:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:27:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:27:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:27:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:27:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:27:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:27:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:27:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:27:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:27:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:27:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:27:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:27:23.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:23 smithi145 ceph-mon[26606]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:23.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:23 smithi145 ceph-mon[26606]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:27:23.873 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:27:23.873 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:27:23.873 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:27:23.873 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:27:23.873 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:27:23.874 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:27:23.874 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:27:23.874 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:27:23.874 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:27:23.874 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:27:23.874 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:27:23.874 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:27:23.874 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:27:23.874 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:27:23.874 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:27:23.874 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:27:23.874 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:27:23.874 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:27:23.875 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:27:23.875 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:27:23.875 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:27:23.875 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:27:23.875 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:27:23.875 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:27:23.875 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:27:23.875 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:27:23.875 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:27:23.875 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:27:23.875 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:27:23.875 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:27:23.875 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:27:23.875 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:27:23.875 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:27:23.876 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:27:23.876 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:27:23.876 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:27:23.876 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:27:23.876 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:27:23.876 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:27:23.876 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:27:23.876 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:27:23.876 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:27:23.876 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:27:23.876 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:27:23.876 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:27:23.876 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:27:23.876 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:27:23.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:27:23.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:27:23.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:27:23.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:27:23.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:27:23.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:27:23.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:27:23.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:27:23.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:27:23.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:27:23.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:27:23.877 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:23 smithi145 ceph-mon[21899]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:23.878 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:23 smithi145 ceph-mon[21899]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:27:23.878 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:27:23.878 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:27:23.878 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:27:23.878 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:27:23.878 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:27:23.878 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:27:23.878 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:27:23.878 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:27:23.878 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:27:23.878 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:27:23.878 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:27:23.878 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:27:23.878 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:27:23.879 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:27:23.879 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:27:23.879 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:27:23.879 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:27:23.879 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:27:23.879 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:27:23.879 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:27:23.879 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:27:23.879 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:27:23.879 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:27:23.879 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:27:23.879 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:27:23.879 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:27:23.879 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:27:23.880 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:27:23.880 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:27:23.880 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:27:23.880 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:27:23.880 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:27:23.880 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:27:23.880 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:27:23.880 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:27:23.880 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:27:23.880 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:27:23.880 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:27:23.880 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:27:23.880 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:27:23.880 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:27:23.880 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:27:23.880 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:27:23.880 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:27:23.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:27:23.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:27:23.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:27:23.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:27:23.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:27:23.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:27:23.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:27:23.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:27:23.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:27:23.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:27:23.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:27:23.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:27:23.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:27:23.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:27:25.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:25 smithi145 ceph-mon[26606]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:25.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:25 smithi145 ceph-mon[21899]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:25.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:27:25] "GET /metrics HTTP/1.1" 200 37998 "" "Prometheus/2.33.4" 2024-04-22T15:27:25.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:25 smithi172 ceph-mon[22747]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:27.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:27 smithi172 ceph-mon[22747]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:27.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:27 smithi145 ceph-mon[26606]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:27.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:27 smithi145 ceph-mon[21899]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:29.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:29 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:27:29.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:29 smithi172 ceph-mon[22747]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:29.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:29 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:27:29.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:29 smithi145 ceph-mon[26606]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:29.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:29 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:27:29.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:29 smithi145 ceph-mon[21899]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:31.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:31 smithi172 ceph-mon[22747]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:31.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:31 smithi145 ceph-mon[26606]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:31.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:31 smithi145 ceph-mon[21899]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:33.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:33 smithi172 ceph-mon[22747]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:33.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:33 smithi145 ceph-mon[21899]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:33.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:33 smithi145 ceph-mon[26606]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:35.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:35 smithi145 ceph-mon[21899]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:35.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:35 smithi145 ceph-mon[26606]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:35.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:27:35] "GET /metrics HTTP/1.1" 200 37998 "" "Prometheus/2.33.4" 2024-04-22T15:27:35.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:35 smithi172 ceph-mon[22747]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:37.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:37 smithi172 ceph-mon[22747]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:37.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:37 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:27:37.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:37 smithi145 ceph-mon[21899]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:37.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:37 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:27:37.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:37 smithi145 ceph-mon[26606]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:37.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:37 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:27:39.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:39 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:27:39.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:39 smithi172 ceph-mon[22747]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:39.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:39 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:27:39.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:39 smithi145 ceph-mon[21899]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:39.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:39 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:27:39.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:39 smithi145 ceph-mon[26606]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:41.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:41 smithi172 ceph-mon[22747]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:41.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:41 smithi145 ceph-mon[21899]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:41.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:41 smithi145 ceph-mon[26606]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:43.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:43 smithi172 ceph-mon[22747]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:43.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:43 smithi145 ceph-mon[21899]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:43.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:43 smithi145 ceph-mon[26606]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:45.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:45 smithi145 ceph-mon[21899]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:45.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:45 smithi145 ceph-mon[26606]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:45.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:27:45] "GET /metrics HTTP/1.1" 200 37991 "" "Prometheus/2.33.4" 2024-04-22T15:27:45.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:45 smithi172 ceph-mon[22747]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:47 smithi172 ceph-mon[22747]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:47.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:47 smithi145 ceph-mon[21899]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:47.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:47 smithi145 ceph-mon[26606]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:49 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:27:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:49 smithi172 ceph-mon[22747]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:49.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:49 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:27:49.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:49 smithi145 ceph-mon[21899]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:49.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:49 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:27:49.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:49 smithi145 ceph-mon[26606]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:51.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:51 smithi172 ceph-mon[22747]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:51.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:51 smithi145 ceph-mon[21899]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:51.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:51 smithi145 ceph-mon[26606]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:52.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:52 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:27:52.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:52 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:27:52.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:52 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:27:53.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:53 smithi172 ceph-mon[22747]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:53.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:53 smithi145 ceph-mon[21899]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:53.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:53 smithi145 ceph-mon[26606]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:55.530 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:55 smithi145 ceph-mon[26606]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:55.531 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:55 smithi145 ceph-mon[21899]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:55.531 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:27:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:27:55] "GET /metrics HTTP/1.1" 200 37997 "" "Prometheus/2.33.4" 2024-04-22T15:27:55.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:55 smithi172 ceph-mon[22747]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:56.154 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch ps' 2024-04-22T15:27:57.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:57 smithi172 ceph-mon[22747]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:57.768 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:57 smithi145 ceph-mon[21899]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:57.768 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:57 smithi145 ceph-mon[26606]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:27:58.099 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:27:58.099 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (9m) 2m ago 17m 25.1M - 0.23.0 ba2b418f427c 2b762f668fa7 2024-04-22T15:27:58.099 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (9m) 2m ago 16m 52.8M - 8.3.5 dad864ee21e9 20a9f6cd1859 2024-04-22T15:27:58.099 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 running (6m) 2m ago 16m 49.0M - 3.5 e1d6a67b021e b59d1eae5fb0 2024-04-22T15:27:58.100 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443,9283 running (11m) 2m ago 21m 448M - 19.0.0-3223-g5883d494 3d983c26e278 1920e9785cbb 2024-04-22T15:27:58.100 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (6m) 2m ago 23m 478M - 19.0.0-3223-g5883d494 3d983c26e278 b2de97566cf4 2024-04-22T15:27:58.100 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (23m) 2m ago 23m 64.3M 2048M 17.2.0 e1d6a67b021e 28106a7bf16e 2024-04-22T15:27:58.100 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (21m) 2m ago 21m 46.1M 2048M 17.2.0 e1d6a67b021e a2fc3297953a 2024-04-22T15:27:58.100 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (22m) 2m ago 22m 45.7M 2048M 17.2.0 e1d6a67b021e b71d9b9f8509 2024-04-22T15:27:58.100 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (9m) 2m ago 17m 15.6M - 1.3.1 1dbe0e931976 7a7a9854f247 2024-04-22T15:27:58.100 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (9m) 2m ago 17m 18.8M - 1.3.1 1dbe0e931976 e5b0a512c835 2024-04-22T15:27:58.100 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (21m) 2m ago 21m 55.2M 2992M 17.2.0 e1d6a67b021e ea9cdc639916 2024-04-22T15:27:58.100 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (20m) 2m ago 20m 53.5M 2992M 17.2.0 e1d6a67b021e 3b9b37a8781f 2024-04-22T15:27:58.100 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (20m) 2m ago 20m 50.4M 2992M 17.2.0 e1d6a67b021e d9502dded8f8 2024-04-22T15:27:58.100 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (20m) 2m ago 20m 51.5M 2992M 17.2.0 e1d6a67b021e 8c72f30e54fc 2024-04-22T15:27:58.100 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (19m) 2m ago 19m 55.3M 3248M 17.2.0 e1d6a67b021e 80001f97ed35 2024-04-22T15:27:58.100 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (19m) 2m ago 19m 53.3M 3248M 17.2.0 e1d6a67b021e ec3ab1335b5c 2024-04-22T15:27:58.101 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (19m) 2m ago 19m 52.6M 3248M 17.2.0 e1d6a67b021e 45d475edf1ca 2024-04-22T15:27:58.101 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (18m) 2m ago 18m 52.2M 3248M 17.2.0 e1d6a67b021e cc9e1d9e7916 2024-04-22T15:27:58.101 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (2m) 2m ago 17m 50.5M - 2.33.4 514e6a882f6e 6e956501159b 2024-04-22T15:27:58.101 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (16m) 2m ago 16m 75.7M - 17.2.0 e1d6a67b021e c90a636c947e 2024-04-22T15:27:58.101 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (16m) 2m ago 16m 73.8M - 17.2.0 e1d6a67b021e 20ac46e9bfda 2024-04-22T15:27:58.697 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph versions' 2024-04-22T15:27:59.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:59 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:27:59.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:59 smithi145 ceph-mon[21899]: from='client.15345 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:27:59.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:27:59 smithi145 ceph-mon[21899]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:59.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:59 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:27:59.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:59 smithi145 ceph-mon[26606]: from='client.15345 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:27:59.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:27:59 smithi145 ceph-mon[26606]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:27:59.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:59 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:27:59.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:59 smithi172 ceph-mon[22747]: from='client.15345 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:27:59.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:27:59 smithi172 ceph-mon[22747]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:28:00.683 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:28:00.683 INFO:teuthology.orchestra.run.smithi145.stdout: "mon": { 2024-04-22T15:28:00.684 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-22T15:28:00.684 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:28:00.684 INFO:teuthology.orchestra.run.smithi145.stdout: "mgr": { 2024-04-22T15:28:00.684 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:28:00.684 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:28:00.684 INFO:teuthology.orchestra.run.smithi145.stdout: "osd": { 2024-04-22T15:28:00.684 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T15:28:00.684 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:28:00.684 INFO:teuthology.orchestra.run.smithi145.stdout: "mds": {}, 2024-04-22T15:28:00.684 INFO:teuthology.orchestra.run.smithi145.stdout: "rgw": { 2024-04-22T15:28:00.684 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:28:00.684 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:28:00.684 INFO:teuthology.orchestra.run.smithi145.stdout: "overall": { 2024-04-22T15:28:00.684 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-22T15:28:00.684 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:28:00.684 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:28:00.685 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:28:01.361 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph -s' 2024-04-22T15:28:01.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:01 smithi145 ceph-mon[21899]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:28:01.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:01 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3781847993' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:28:01.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:01 smithi145 ceph-mon[26606]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:28:01.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:01 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3781847993' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:28:01.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:01 smithi172 ceph-mon[22747]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:28:01.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:01 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/3781847993' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:28:03.449 INFO:teuthology.orchestra.run.smithi145.stdout: cluster: 2024-04-22T15:28:03.449 INFO:teuthology.orchestra.run.smithi145.stdout: id: 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:28:03.449 INFO:teuthology.orchestra.run.smithi145.stdout: health: HEALTH_OK 2024-04-22T15:28:03.449 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-04-22T15:28:03.450 INFO:teuthology.orchestra.run.smithi145.stdout: services: 2024-04-22T15:28:03.450 INFO:teuthology.orchestra.run.smithi145.stdout: mon: 3 daemons, quorum a,c,b (age 21m) 2024-04-22T15:28:03.450 INFO:teuthology.orchestra.run.smithi145.stdout: mgr: y(active, since 3m), standbys: x 2024-04-22T15:28:03.450 INFO:teuthology.orchestra.run.smithi145.stdout: osd: 8 osds: 8 up (since 18m), 8 in (since 18m) 2024-04-22T15:28:03.450 INFO:teuthology.orchestra.run.smithi145.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-04-22T15:28:03.450 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-04-22T15:28:03.450 INFO:teuthology.orchestra.run.smithi145.stdout: data: 2024-04-22T15:28:03.450 INFO:teuthology.orchestra.run.smithi145.stdout: pools: 6 pools, 161 pgs 2024-04-22T15:28:03.450 INFO:teuthology.orchestra.run.smithi145.stdout: objects: 209 objects, 585 KiB 2024-04-22T15:28:03.450 INFO:teuthology.orchestra.run.smithi145.stdout: usage: 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:28:03.450 INFO:teuthology.orchestra.run.smithi145.stdout: pgs: 161 active+clean 2024-04-22T15:28:03.450 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-04-22T15:28:03.450 INFO:teuthology.orchestra.run.smithi145.stdout: io: 2024-04-22T15:28:03.450 INFO:teuthology.orchestra.run.smithi145.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-04-22T15:28:03.450 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-04-22T15:28:03.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:03 smithi172 ceph-mon[22747]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:28:03.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:03 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/4129158666' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T15:28:03.776 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:03 smithi145 ceph-mon[26606]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:28:03.776 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:03 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/4129158666' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T15:28:03.777 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:03 smithi145 ceph-mon[21899]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:28:03.777 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:03 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/4129158666' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T15:28:04.084 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph health detail' 2024-04-22T15:28:05.521 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:05 smithi145 ceph-mon[21899]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:28:05.521 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:05 smithi145 ceph-mon[26606]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:28:05.521 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:05 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:28:05] "GET /metrics HTTP/1.1" 200 37997 "" "Prometheus/2.33.4" 2024-04-22T15:28:05.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:05 smithi172 ceph-mon[22747]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:28:06.143 INFO:teuthology.orchestra.run.smithi145.stdout:HEALTH_OK 2024-04-22T15:28:06.508 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:06 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/4072516023' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:28:06.508 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:06 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/4072516023' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:28:06.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:06 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/4072516023' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:28:06.801 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-04-22T15:28:07.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:07 smithi145 ceph-mon[21899]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:28:07.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:07 smithi145 ceph-mon[26606]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:28:07.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:07 smithi172 ceph-mon[22747]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:28:08.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:08 smithi145 ceph-mon[21899]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:28:08.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:08 smithi145 ceph-mon[26606]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:28:08.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:08 smithi172 ceph-mon[22747]: from='mgr.15231 172.21.15.145:0/800247455' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:28:08.879 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:28:09.549 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:09 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:28:09.549 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:09 smithi145 ceph-mon[26606]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:28:09.549 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:09 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/1528547878' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:28:09.549 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:09 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:28:09.549 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:09 smithi145 ceph-mon[21899]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:28:09.549 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:09 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/1528547878' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:28:09.573 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph mgr fail' 2024-04-22T15:28:09.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:09 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:28:09.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:09 smithi172 ceph-mon[22747]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:28:09.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:09 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/1528547878' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:28:11.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:11 smithi172 ceph-mon[22747]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:28:11.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:11 smithi145 ceph-mon[21899]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:28:11.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:11 smithi145 ceph-mon[26606]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:28:12.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:12 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/830224262' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-22T15:28:12.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:12 smithi172 ceph-mon[22747]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-22T15:28:12.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:12 smithi172 ceph-mon[22747]: osdmap e106: 8 total, 8 up, 8 in 2024-04-22T15:28:12.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:12 smithi172 ceph-mon[22747]: Standby manager daemon y started 2024-04-22T15:28:12.718 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:28:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:28:12] ENGINE Bus STOPPING 2024-04-22T15:28:12.719 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:28:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:28:12] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-22T15:28:12.719 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:28:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:28:12] ENGINE Bus STOPPED 2024-04-22T15:28:12.719 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:28:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:28:12] ENGINE Bus STARTING 2024-04-22T15:28:12.719 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:28:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:28:12] ENGINE Serving on http://:::9283 2024-04-22T15:28:12.719 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:28:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:28:12] ENGINE Bus STARTED 2024-04-22T15:28:12.830 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:12 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/830224262' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-22T15:28:12.831 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:12 smithi145 ceph-mon[21899]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-22T15:28:12.831 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:12 smithi145 ceph-mon[21899]: osdmap e106: 8 total, 8 up, 8 in 2024-04-22T15:28:12.831 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:12 smithi145 ceph-mon[21899]: Standby manager daemon y started 2024-04-22T15:28:12.831 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:12 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/830224262' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-22T15:28:12.831 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:12 smithi145 ceph-mon[26606]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-22T15:28:12.831 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:12 smithi145 ceph-mon[26606]: osdmap e106: 8 total, 8 up, 8 in 2024-04-22T15:28:12.831 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:12 smithi145 ceph-mon[26606]: Standby manager daemon y started 2024-04-22T15:28:12.831 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ignoring --setuser ceph since I am not root 2024-04-22T15:28:12.831 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ignoring --setgroup ceph since I am not root 2024-04-22T15:28:12.831 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:12 smithi145 ceph-mgr[60909]: -- 172.21.15.145:0/2876699081 <== mon.1 v2:172.21.15.145:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x564b400ca5a0 con 0x564b4009e800 2024-04-22T15:28:12.832 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:28:12.646+0000 7f43234b5200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T15:28:13.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:28:12.841+0000 7f43234b5200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T15:28:13.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:28:12.951+0000 7f43234b5200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T15:28:13.196 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'sleep 180' 2024-04-22T15:28:13.471 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:28:13.288+0000 7f43234b5200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T15:28:13.471 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:28:13.388+0000 7f43234b5200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T15:28:13.799 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:13 smithi172 ceph-mon[22747]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-22T15:28:13.799 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:13 smithi172 ceph-mon[22747]: mgrmap e33: x(active, starting, since 0.867938s), standbys: y 2024-04-22T15:28:13.799 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:13 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:28:13.799 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:13 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:28:13.800 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:13 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:28:13.800 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:13 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T15:28:13.800 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:13 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T15:28:13.800 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:13 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:28:13.800 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:13 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:28:13.800 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:13 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:28:13.800 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:13 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:28:13.800 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:13 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:28:13.800 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:13 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:28:13.800 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:13 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:28:13.800 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:13 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:28:13.800 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:13 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:28:13.800 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:13 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:28:13.800 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:13 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:28:13.801 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:13 smithi172 ceph-mon[22747]: Manager daemon x is now available 2024-04-22T15:28:13.801 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:13 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:28:13.801 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:13 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:28:13.801 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:13 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:28:13.801 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:13 smithi172 ceph-mon[22747]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:28:13.801 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:13 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-22T15:28:13.801 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:13 smithi172 ceph-mon[22747]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-22T15:28:13.801 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:13 smithi172 ceph-mon[22747]: [22/Apr/2024:15:28:13] ENGINE Bus STARTING 2024-04-22T15:28:13.801 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:13 smithi172 ceph-mon[22747]: [22/Apr/2024:15:28:13] ENGINE Serving on https://172.21.15.172:7150 2024-04-22T15:28:13.849 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[21899]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-22T15:28:13.849 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[21899]: mgrmap e33: x(active, starting, since 0.867938s), standbys: y 2024-04-22T15:28:13.850 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:28:13.850 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:28:13.850 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:28:13.850 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T15:28:13.850 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T15:28:13.850 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:28:13.850 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:28:13.850 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:28:13.850 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:28:13.850 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:28:13.850 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:28:13.850 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:28:13.850 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:28:13.850 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:28:13.851 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:28:13.851 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:28:13.851 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[21899]: Manager daemon x is now available 2024-04-22T15:28:13.851 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:28:13.851 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:28:13.851 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:28:13.851 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[21899]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:28:13.851 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-22T15:28:13.851 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[21899]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-22T15:28:13.851 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[21899]: [22/Apr/2024:15:28:13] ENGINE Bus STARTING 2024-04-22T15:28:13.851 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[21899]: [22/Apr/2024:15:28:13] ENGINE Serving on https://172.21.15.172:7150 2024-04-22T15:28:13.851 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[26606]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-22T15:28:13.851 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[26606]: mgrmap e33: x(active, starting, since 0.867938s), standbys: y 2024-04-22T15:28:13.852 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:28:13.852 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:28:13.852 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:28:13.852 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T15:28:13.852 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T15:28:13.852 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:28:13.852 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:28:13.852 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:28:13.852 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:28:13.852 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:28:13.852 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:28:13.852 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:28:13.852 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:28:13.852 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:28:13.852 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:28:13.852 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:28:13.853 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[26606]: Manager daemon x is now available 2024-04-22T15:28:13.853 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:28:13.853 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:28:13.853 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:28:13.853 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[26606]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:28:13.853 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-22T15:28:13.853 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[26606]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-22T15:28:13.853 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[26606]: [22/Apr/2024:15:28:13] ENGINE Bus STARTING 2024-04-22T15:28:13.853 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:13 smithi145 ceph-mon[26606]: [22/Apr/2024:15:28:13] ENGINE Serving on https://172.21.15.172:7150 2024-04-22T15:28:14.673 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:14 smithi145 ceph-mon[21899]: mgrmap e34: x(active, since 1.86716s), standbys: y 2024-04-22T15:28:14.673 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:14 smithi145 ceph-mon[21899]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:28:14.673 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:14 smithi145 ceph-mon[21899]: [22/Apr/2024:15:28:13] ENGINE Serving on http://172.21.15.172:8765 2024-04-22T15:28:14.673 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:14 smithi145 ceph-mon[21899]: [22/Apr/2024:15:28:13] ENGINE Bus STARTED 2024-04-22T15:28:14.673 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:14 smithi145 ceph-mon[26606]: mgrmap e34: x(active, since 1.86716s), standbys: y 2024-04-22T15:28:14.673 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:14 smithi145 ceph-mon[26606]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:28:14.673 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:14 smithi145 ceph-mon[26606]: [22/Apr/2024:15:28:13] ENGINE Serving on http://172.21.15.172:8765 2024-04-22T15:28:14.673 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:14 smithi145 ceph-mon[26606]: [22/Apr/2024:15:28:13] ENGINE Bus STARTED 2024-04-22T15:28:14.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:14 smithi172 ceph-mon[22747]: mgrmap e34: x(active, since 1.86716s), standbys: y 2024-04-22T15:28:14.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:14 smithi172 ceph-mon[22747]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:28:14.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:14 smithi172 ceph-mon[22747]: [22/Apr/2024:15:28:13] ENGINE Serving on http://172.21.15.172:8765 2024-04-22T15:28:14.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:14 smithi172 ceph-mon[22747]: [22/Apr/2024:15:28:13] ENGINE Bus STARTED 2024-04-22T15:28:15.024 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:14 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:28:14.669+0000 7f43234b5200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T15:28:15.024 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:14 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:28:14.846+0000 7f43234b5200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T15:28:15.025 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:28:15.021+0000 7f43234b5200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T15:28:15.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:15 smithi145 ceph-mon[21899]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:28:15.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:15 smithi145 ceph-mon[21899]: mgrmap e35: x(active, since 2s), standbys: y 2024-04-22T15:28:15.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:28:15.325+0000 7f43234b5200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T15:28:15.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:28:15.420+0000 7f43234b5200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T15:28:15.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:15 smithi145 ceph-mon[26606]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:28:15.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:15 smithi145 ceph-mon[26606]: mgrmap e35: x(active, since 2s), standbys: y 2024-04-22T15:28:15.647 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:15 smithi172 ceph-mon[22747]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:28:15.647 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:15 smithi172 ceph-mon[22747]: mgrmap e35: x(active, since 2s), standbys: y 2024-04-22T15:28:15.914 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:28:15.637+0000 7f43234b5200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T15:28:15.914 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:28:15.854+0000 7f43234b5200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T15:28:16.622 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:16 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:28:16.493+0000 7f43234b5200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T15:28:17.122 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:16 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:28:16.962+0000 7f43234b5200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T15:28:17.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:28:17.064+0000 7f43234b5200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T15:28:17.605 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:28:17.275+0000 7f43234b5200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-22T15:28:17.605 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:17 smithi145 ceph-mon[26606]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:28:17.605 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:17 smithi145 ceph-mon[26606]: mgrmap e36: x(active, since 4s), standbys: y 2024-04-22T15:28:17.605 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:17 smithi145 ceph-mon[26606]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:17.605 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:17 smithi145 ceph-mon[26606]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:17.605 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:17 smithi145 ceph-mon[21899]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:28:17.606 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:17 smithi145 ceph-mon[21899]: mgrmap e36: x(active, since 4s), standbys: y 2024-04-22T15:28:17.606 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:17 smithi145 ceph-mon[21899]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:17.606 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:17 smithi145 ceph-mon[21899]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:17.872 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:28:17.830+0000 7f43234b5200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T15:28:17.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:17 smithi172 ceph-mon[22747]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:28:17.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:17 smithi172 ceph-mon[22747]: mgrmap e36: x(active, since 4s), standbys: y 2024-04-22T15:28:17.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:17 smithi172 ceph-mon[22747]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:17.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:17 smithi172 ceph-mon[22747]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:18.199 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:28:17.927+0000 7f43234b5200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T15:28:18.552 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:18 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:28:18.196+0000 7f43234b5200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T15:28:18.552 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:18 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:28:18.335+0000 7f43234b5200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T15:28:18.552 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:18 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:28:18.441+0000 7f43234b5200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T15:28:18.872 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:18 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:28:18.572+0000 7f43234b5200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T15:28:19.331 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:18 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:28:18.884+0000 7f43234b5200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T15:28:19.583 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:19 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:28:19.583 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:19 smithi145 ceph-mon[26606]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:19.583 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:19 smithi145 ceph-mon[26606]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:19.583 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:19 smithi145 ceph-mon[26606]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:19.583 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:19 smithi145 ceph-mon[26606]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:19.584 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:19 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:28:19.584 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:19 smithi145 ceph-mon[21899]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:19.584 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:19 smithi145 ceph-mon[21899]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:19.584 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:19 smithi145 ceph-mon[21899]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:19.584 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:19 smithi145 ceph-mon[21899]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:19.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:19 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:28:19.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:19 smithi172 ceph-mon[22747]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:19.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:19 smithi172 ceph-mon[22747]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:19.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:19 smithi172 ceph-mon[22747]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:19.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:19 smithi172 ceph-mon[22747]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:19.872 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:28:19.580+0000 7f43234b5200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T15:28:19.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:28:19.819+0000 7f43234b5200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T15:28:19.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:28:19] ENGINE Bus STARTING 2024-04-22T15:28:20.339 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CherryPy Checker: 2024-04-22T15:28:20.339 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: The Application mounted at '' has an empty config. 2024-04-22T15:28:20.339 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:28:20.340 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:28:19] ENGINE Serving on http://:::9283 2024-04-22T15:28:20.340 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:28:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:28:19] ENGINE Bus STARTED 2024-04-22T15:28:20.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:20 smithi145 ceph-mon[21899]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:28:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:20 smithi145 ceph-mon[21899]: Standby manager daemon y restarted 2024-04-22T15:28:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:20 smithi145 ceph-mon[21899]: Standby manager daemon y started 2024-04-22T15:28:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:20 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-22T15:28:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:20 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:28:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:20 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-22T15:28:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:20 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:28:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:20 smithi145 ceph-mon[21899]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:20 smithi145 ceph-mon[21899]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:20 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:28:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:20 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:28:20.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:20 smithi145 ceph-mon[26606]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:28:20.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:20 smithi145 ceph-mon[26606]: Standby manager daemon y restarted 2024-04-22T15:28:20.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:20 smithi145 ceph-mon[26606]: Standby manager daemon y started 2024-04-22T15:28:20.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:20 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-22T15:28:20.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:20 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:28:20.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:20 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-22T15:28:20.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:20 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:28:20.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:20 smithi145 ceph-mon[26606]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:20.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:20 smithi145 ceph-mon[26606]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:20.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:20 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:28:20.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:20 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:28:20.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:20 smithi172 ceph-mon[22747]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:28:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:20 smithi172 ceph-mon[22747]: Standby manager daemon y restarted 2024-04-22T15:28:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:20 smithi172 ceph-mon[22747]: Standby manager daemon y started 2024-04-22T15:28:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:20 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-22T15:28:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:20 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:28:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:20 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-22T15:28:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:20 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:28:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:20 smithi172 ceph-mon[22747]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:20 smithi172 ceph-mon[22747]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:20 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:28:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:20 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:28:21.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:21 smithi145 ceph-mon[21899]: Updating smithi145:/etc/ceph/ceph.conf 2024-04-22T15:28:21.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:21 smithi145 ceph-mon[21899]: Updating smithi172:/etc/ceph/ceph.conf 2024-04-22T15:28:21.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:21 smithi145 ceph-mon[21899]: Updating smithi145:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:28:21.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:21 smithi145 ceph-mon[21899]: Updating smithi172:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:28:21.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:21 smithi145 ceph-mon[21899]: mgrmap e37: x(active, since 8s), standbys: y 2024-04-22T15:28:21.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:21 smithi145 ceph-mon[21899]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:21.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:21 smithi145 ceph-mon[21899]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:21.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:21 smithi145 ceph-mon[21899]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:21.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:21 smithi145 ceph-mon[21899]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:21.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:21 smithi145 ceph-mon[21899]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:21.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:21 smithi145 ceph-mon[26606]: Updating smithi145:/etc/ceph/ceph.conf 2024-04-22T15:28:21.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:21 smithi145 ceph-mon[26606]: Updating smithi172:/etc/ceph/ceph.conf 2024-04-22T15:28:21.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:21 smithi145 ceph-mon[26606]: Updating smithi145:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:28:21.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:21 smithi145 ceph-mon[26606]: Updating smithi172:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:28:21.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:21 smithi145 ceph-mon[26606]: mgrmap e37: x(active, since 8s), standbys: y 2024-04-22T15:28:21.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:21 smithi145 ceph-mon[26606]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:21.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:21 smithi145 ceph-mon[26606]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:21.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:21 smithi145 ceph-mon[26606]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:21.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:21 smithi145 ceph-mon[26606]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:21.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:21 smithi145 ceph-mon[26606]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:21.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:21 smithi172 ceph-mon[22747]: Updating smithi145:/etc/ceph/ceph.conf 2024-04-22T15:28:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:21 smithi172 ceph-mon[22747]: Updating smithi172:/etc/ceph/ceph.conf 2024-04-22T15:28:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:21 smithi172 ceph-mon[22747]: Updating smithi145:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:28:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:21 smithi172 ceph-mon[22747]: Updating smithi172:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:28:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:21 smithi172 ceph-mon[22747]: mgrmap e37: x(active, since 8s), standbys: y 2024-04-22T15:28:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:21 smithi172 ceph-mon[22747]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:21 smithi172 ceph-mon[22747]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:21 smithi172 ceph-mon[22747]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:21 smithi172 ceph-mon[22747]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:21 smithi172 ceph-mon[22747]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:22.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:22 smithi145 ceph-mon[26606]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-22T15:28:22.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:22 smithi145 ceph-mon[26606]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:28:22.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:22 smithi145 ceph-mon[26606]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:28:22.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:22 smithi145 ceph-mon[26606]: Updating smithi145:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:28:22.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:22 smithi145 ceph-mon[26606]: Updating smithi172:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:28:22.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:22 smithi145 ceph-mon[26606]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-22T15:28:22.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:22 smithi145 ceph-mon[26606]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T15:28:22.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:22 smithi145 ceph-mon[26606]: Reconfiguring daemon prometheus.a on smithi172 2024-04-22T15:28:22.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:22 smithi145 ceph-mon[26606]: Health check failed: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in error state'] (CEPHADM_FAILED_DAEMON) 2024-04-22T15:28:22.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:22 smithi145 ceph-mon[21899]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-22T15:28:22.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:22 smithi145 ceph-mon[21899]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:28:22.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:22 smithi145 ceph-mon[21899]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:28:22.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:22 smithi145 ceph-mon[21899]: Updating smithi145:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:28:22.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:22 smithi145 ceph-mon[21899]: Updating smithi172:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:28:22.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:22 smithi145 ceph-mon[21899]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-22T15:28:22.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:22 smithi145 ceph-mon[21899]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T15:28:22.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:22 smithi145 ceph-mon[21899]: Reconfiguring daemon prometheus.a on smithi172 2024-04-22T15:28:22.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:22 smithi145 ceph-mon[21899]: Health check failed: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in error state'] (CEPHADM_FAILED_DAEMON) 2024-04-22T15:28:22.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:22 smithi172 ceph-mon[22747]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-22T15:28:22.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:22 smithi172 ceph-mon[22747]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:28:22.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:22 smithi172 ceph-mon[22747]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:28:22.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:22 smithi172 ceph-mon[22747]: Updating smithi145:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:28:22.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:22 smithi172 ceph-mon[22747]: Updating smithi172:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:28:22.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:22 smithi172 ceph-mon[22747]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-22T15:28:22.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:22 smithi172 ceph-mon[22747]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T15:28:22.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:22 smithi172 ceph-mon[22747]: Reconfiguring daemon prometheus.a on smithi172 2024-04-22T15:28:22.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:22 smithi172 ceph-mon[22747]: Health check failed: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in error state'] (CEPHADM_FAILED_DAEMON) 2024-04-22T15:28:24.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:24 smithi145 ceph-mon[21899]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-22T15:28:24.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:24 smithi145 ceph-mon[26606]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-22T15:28:24.670 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:24 smithi172 ceph-mon[22747]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-22T15:28:24.671 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:24 smithi172 systemd[1]: Stopping Ceph prometheus.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:28:24.671 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:24 smithi172 bash[54543]: Error: no container with name or ID "ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-22T15:28:24.968 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:24 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:28:24.667Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-22T15:28:24.968 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:24 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:28:24.668Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-22T15:28:24.968 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:24 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:28:24.668Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-22T15:28:24.968 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:24 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:28:24.668Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-22T15:28:24.968 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:24 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:28:24.668Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-22T15:28:24.968 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:24 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:28:24.668Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-22T15:28:24.968 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:24 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:28:24.668Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-22T15:28:24.968 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:24 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:28:24.668Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-22T15:28:24.968 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:24 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:28:24.668Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-22T15:28:24.969 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:24 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:28:24.668Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-22T15:28:24.969 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:24 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:28:24.668Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-22T15:28:24.969 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:24 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[51641]: ts=2024-04-22T15:28:24.668Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-22T15:28:24.969 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:24 smithi172 podman[54556]: 2024-04-22 15:28:24.678648499 +0000 UTC m=+0.140201879 container died 6e956501159b424eba9e689e286ddcf241d763c8e30e7207adec753ca2468d4c (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:28:25.310 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:25 smithi172 podman[54556]: 2024-04-22 15:28:25.065492628 +0000 UTC m=+0.527046045 container cleanup 6e956501159b424eba9e689e286ddcf241d763c8e30e7207adec753ca2468d4c (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:28:25.310 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:25 smithi172 bash[54556]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a 2024-04-22T15:28:25.310 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:25 smithi172 bash[54586]: Error: no container with name or ID "ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-22T15:28:25.570 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:25 smithi172 podman[54570]: 2024-04-22 15:28:25.308043183 +0000 UTC m=+0.628560438 container remove 6e956501159b424eba9e689e286ddcf241d763c8e30e7207adec753ca2468d4c (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:28:25.570 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:25 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-22T15:28:25.570 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:25 smithi172 systemd[1]: Stopped Ceph prometheus.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:28:25.570 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:25 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.398s CPU time. 2024-04-22T15:28:25.570 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:25 smithi172 systemd[1]: Starting Ceph prometheus.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:28:25.570 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:25 smithi172 podman[54656]: 2024-04-22 15:28:25.442237376 +0000 UTC m=+0.018981418 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-22T15:28:25.570 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:25 smithi172 podman[54656]: 2024-04-22 15:28:25.567728279 +0000 UTC m=+0.144472289 container create 6d94243c82c3ccb9b4cd7166c007b278473eed869bf9143a8cdede7f4fa90d4f (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:28:25.864 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:25 smithi172 podman[54656]: 2024-04-22 15:28:25.786541184 +0000 UTC m=+0.363285198 container init 6d94243c82c3ccb9b4cd7166c007b278473eed869bf9143a8cdede7f4fa90d4f (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:28:25.864 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:25 smithi172 podman[54656]: 2024-04-22 15:28:25.789365916 +0000 UTC m=+0.366109932 container start 6d94243c82c3ccb9b4cd7166c007b278473eed869bf9143a8cdede7f4fa90d4f (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:28:25.864 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:25 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:28:25.812Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-22T15:28:25.864 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:25 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:28:25.812Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-22T15:28:25.864 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:25 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:28:25.812Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-22T15:28:25.864 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:25 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:28:25.812Z caller=main.go:518 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi172 (none))" 2024-04-22T15:28:25.865 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:25 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:28:25.812Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-22T15:28:25.865 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:25 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:28:25.812Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-22T15:28:25.865 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:25 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:28:25.814Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-22T15:28:25.865 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:25 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:28:25.815Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-22T15:28:25.865 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:25 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:28:25.815Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-22T15:28:25.865 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:25 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:28:25.817Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-22T15:28:25.865 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:25 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:28:25.817Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.439µs 2024-04-22T15:28:25.865 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:25 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:28:25.817Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-22T15:28:25.865 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:25 smithi172 bash[54656]: 6d94243c82c3ccb9b4cd7166c007b278473eed869bf9143a8cdede7f4fa90d4f 2024-04-22T15:28:26.217 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:25 smithi172 systemd[1]: Started Ceph prometheus.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:28:26.218 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:26 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:28:26.024Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=4 2024-04-22T15:28:26.498 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:26 smithi172 ceph-mon[22747]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T15:28:26.498 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:26 smithi172 ceph-mon[22747]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:26.499 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:26 smithi172 ceph-mon[22747]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:26.499 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:26 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:28:26.499 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:26 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:28:26.499 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:26 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:28:26.241Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=4 2024-04-22T15:28:26.499 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:26 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:28:26.460Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=4 2024-04-22T15:28:26.499 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:26 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:28:26.463Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=4 2024-04-22T15:28:26.499 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:26 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:28:26.463Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=4 2024-04-22T15:28:26.499 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:26 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:28:26.463Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=33.78µs wal_replay_duration=646.631377ms total_replay_duration=646.67723ms 2024-04-22T15:28:26.499 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:26 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:28:26.469Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-22T15:28:26.499 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:26 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:28:26.469Z caller=main.go:947 level=info msg="TSDB started" 2024-04-22T15:28:26.499 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:26 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:28:26.469Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-22T15:28:26.499 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:26 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:28:26.495Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=26.334947ms db_storage=792ns remote_storage=1.273µs web_handler=347ns query_engine=914ns scrape=155.36µs scrape_sd=113.27µs notify=10.011µs notify_sd=12.464µs rules=25.705359ms 2024-04-22T15:28:26.500 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:28:26 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:28:26.496Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-22T15:28:26.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:26 smithi145 ceph-mon[21899]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T15:28:26.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:26 smithi145 ceph-mon[21899]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:26.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:26 smithi145 ceph-mon[21899]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:26.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:26 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:28:26.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:26 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:28:26.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:26 smithi145 ceph-mon[26606]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T15:28:26.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:26 smithi145 ceph-mon[26606]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:26.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:26 smithi145 ceph-mon[26606]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:26.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:26 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:28:26.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:26 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:28:27.611 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:27 smithi172 ceph-mon[22747]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:28:27.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:27 smithi145 ceph-mon[21899]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:28:27.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:27 smithi145 ceph-mon[26606]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:28:28.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:28 smithi172 ceph-mon[22747]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T15:28:28.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:28 smithi172 ceph-mon[22747]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:28.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:28 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:28:28.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:28 smithi145 ceph-mon[21899]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T15:28:28.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:28 smithi145 ceph-mon[21899]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:28.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:28 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:28:28.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:28 smithi145 ceph-mon[26606]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T15:28:28.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:28 smithi145 ceph-mon[26606]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:28.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:28 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:28:29.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:29 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:28:29.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:29 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:28:29.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:29 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:28:30.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:30 smithi145 ceph-mon[21899]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-22T15:28:30.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:30 smithi145 ceph-mon[21899]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:30.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:30 smithi145 ceph-mon[21899]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:30.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:30 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:28:30.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:30 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:28:30.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:30 smithi145 ceph-mon[21899]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:30.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:30 smithi145 ceph-mon[26606]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-22T15:28:30.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:30 smithi145 ceph-mon[26606]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:30.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:30 smithi145 ceph-mon[26606]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:30.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:30 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:28:30.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:30 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:28:30.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:30 smithi145 ceph-mon[26606]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:30.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:30 smithi172 ceph-mon[22747]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-22T15:28:30.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:30 smithi172 ceph-mon[22747]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:30.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:30 smithi172 ceph-mon[22747]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:30.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:30 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:28:30.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:30 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:28:30.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:30 smithi172 ceph-mon[22747]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:28:31.688 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:31 smithi172 ceph-mon[22747]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:28:31.688 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:31 smithi172 ceph-mon[22747]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:28:31.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:31 smithi145 ceph-mon[21899]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:28:31.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:31 smithi145 ceph-mon[21899]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:28:31.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:31 smithi145 ceph-mon[26606]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:28:31.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:31 smithi145 ceph-mon[26606]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:28:31.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:28:31 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:28:31] "GET /metrics HTTP/1.1" 200 37990 "" "Prometheus/2.33.4" 2024-04-22T15:28:33.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:33 smithi172 ceph-mon[22747]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:28:33.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:33 smithi145 ceph-mon[21899]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:28:33.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:33 smithi145 ceph-mon[26606]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:28:35.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:35 smithi172 ceph-mon[22747]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:28:35.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:35 smithi145 ceph-mon[21899]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:28:35.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:35 smithi145 ceph-mon[26606]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:28:37.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:37 smithi172 ceph-mon[22747]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:28:37.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:37 smithi145 ceph-mon[21899]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:28:37.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:37 smithi145 ceph-mon[26606]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:28:39.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:39 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:28:39.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:39 smithi172 ceph-mon[22747]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 561 B/s rd, 0 op/s 2024-04-22T15:28:39.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:39 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:28:39.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:39 smithi145 ceph-mon[21899]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 561 B/s rd, 0 op/s 2024-04-22T15:28:39.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:39 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:28:39.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:39 smithi145 ceph-mon[26606]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 561 B/s rd, 0 op/s 2024-04-22T15:28:41.686 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:41 smithi172 ceph-mon[22747]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:28:41.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:41 smithi145 ceph-mon[21899]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:28:41.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:41 smithi145 ceph-mon[26606]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:28:41.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:28:41 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:28:41] "GET /metrics HTTP/1.1" 200 37990 "" "Prometheus/2.33.4" 2024-04-22T15:28:43.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:43 smithi172 ceph-mon[22747]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:28:43.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:43 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:28:43.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:43 smithi145 ceph-mon[21899]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:28:43.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:43 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:28:43.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:43 smithi145 ceph-mon[26606]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:28:43.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:43 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:28:45.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:45 smithi172 ceph-mon[22747]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:28:45.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:45 smithi145 ceph-mon[21899]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:28:45.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:45 smithi145 ceph-mon[26606]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:28:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:47 smithi172 ceph-mon[22747]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:28:47.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:47 smithi145 ceph-mon[21899]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:28:47.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:47 smithi145 ceph-mon[26606]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:28:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:49 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:28:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:49 smithi172 ceph-mon[22747]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:28:49.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:49 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:28:49.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:49 smithi145 ceph-mon[21899]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:28:49.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:49 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:28:49.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:49 smithi145 ceph-mon[26606]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:28:51.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:51 smithi145 ceph-mon[21899]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:28:51.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:51 smithi145 ceph-mon[26606]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:28:51.687 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:51 smithi172 ceph-mon[22747]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:28:51.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:28:51 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:28:51] "GET /metrics HTTP/1.1" 200 37990 "" "Prometheus/2.33.4" 2024-04-22T15:28:53.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:53 smithi172 ceph-mon[22747]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:28:53.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:53 smithi145 ceph-mon[26606]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:28:53.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:53 smithi145 ceph-mon[21899]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:28:55.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:55 smithi172 ceph-mon[22747]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:28:55.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:55 smithi145 ceph-mon[26606]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:28:55.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:55 smithi145 ceph-mon[21899]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:28:57.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:57 smithi172 ceph-mon[22747]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:28:57.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:57 smithi145 ceph-mon[26606]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:28:57.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:57 smithi145 ceph-mon[21899]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:28:58.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:58 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:28:58.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:58 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:28:58.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:58 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:28:59.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:59 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:28:59.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:28:59 smithi172 ceph-mon[22747]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:28:59.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:59 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:28:59.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:28:59 smithi145 ceph-mon[21899]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:28:59.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:59 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:28:59.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:28:59 smithi145 ceph-mon[26606]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:01.687 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:01 smithi172 ceph-mon[22747]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:29:01.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:01 smithi145 ceph-mon[21899]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:29:01.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:01 smithi145 ceph-mon[26606]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:29:01.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:29:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:29:01] "GET /metrics HTTP/1.1" 200 37993 "" "Prometheus/2.33.4" 2024-04-22T15:29:03.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:03 smithi172 ceph-mon[22747]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:03.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:03 smithi145 ceph-mon[21899]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:03.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:03 smithi145 ceph-mon[26606]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:05.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:05 smithi172 ceph-mon[22747]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:29:05.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:05 smithi145 ceph-mon[21899]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:29:05.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:05 smithi145 ceph-mon[26606]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:29:06.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:06 smithi145 ceph-mon[21899]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:06.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:06 smithi145 ceph-mon[26606]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:06.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:06 smithi172 ceph-mon[22747]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:09.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:09 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:29:09.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:09 smithi172 ceph-mon[22747]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:09.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:09 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:29:09.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:09 smithi145 ceph-mon[21899]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:09.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:09 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:29:09.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:09 smithi145 ceph-mon[26606]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:11.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:11 smithi145 ceph-mon[21899]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:29:11.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:11 smithi145 ceph-mon[26606]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:29:11.687 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:11 smithi172 ceph-mon[22747]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:29:11.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:29:11 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:29:11] "GET /metrics HTTP/1.1" 200 37993 "" "Prometheus/2.33.4" 2024-04-22T15:29:13.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:13 smithi145 ceph-mon[21899]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:13.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:13 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:29:13.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:13 smithi145 ceph-mon[26606]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:13.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:13 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:29:13.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:13 smithi172 ceph-mon[22747]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:13.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:13 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:29:15.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:15 smithi145 ceph-mon[21899]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:29:15.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:15 smithi145 ceph-mon[26606]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:29:15.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:15 smithi172 ceph-mon[22747]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:29:17.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:17 smithi145 ceph-mon[21899]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:17.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:17 smithi145 ceph-mon[26606]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:17.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:17 smithi172 ceph-mon[22747]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:19.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:19 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:29:19.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:19 smithi145 ceph-mon[21899]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:19.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:19 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:29:19.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:19 smithi145 ceph-mon[26606]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:19.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:19 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:29:19.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:19 smithi172 ceph-mon[22747]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:21.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:21 smithi145 ceph-mon[21899]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:29:21.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:21 smithi145 ceph-mon[26606]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:29:21.687 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:21 smithi172 ceph-mon[22747]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:29:21.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:29:21 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:29:21] "GET /metrics HTTP/1.1" 200 37988 "" "Prometheus/2.33.4" 2024-04-22T15:29:23.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:23 smithi145 ceph-mon[26606]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:23 smithi145 ceph-mon[21899]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:23.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:23 smithi172 ceph-mon[22747]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:25.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:25 smithi145 ceph-mon[21899]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:29:25.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:25 smithi145 ceph-mon[26606]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:29:25.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:25 smithi172 ceph-mon[22747]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:29:27.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:27 smithi145 ceph-mon[21899]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:27.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:27 smithi145 ceph-mon[26606]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:27.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:27 smithi172 ceph-mon[22747]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:28.302 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:28 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:29:28.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:28 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:29:28.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:28 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:29:29.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:29 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:29:29.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:29 smithi145 ceph-mon[21899]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:29.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:29 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:29:29.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:29 smithi145 ceph-mon[26606]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:29.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:29 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:29:29.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:29 smithi172 ceph-mon[22747]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:30.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:30 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:29:30.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:30 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:29:30.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:30 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:29:31.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:31 smithi145 ceph-mon[21899]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:29:31.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:31 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:29:31.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:31 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:29:31.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:31 smithi145 ceph-mon[21899]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:29:31.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:31 smithi145 ceph-mon[26606]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:29:31.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:31 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:29:31.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:31 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:29:31.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:31 smithi145 ceph-mon[26606]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:29:31.686 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:31 smithi172 ceph-mon[22747]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:29:31.686 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:31 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:29:31.686 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:31 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:29:31.686 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:31 smithi172 ceph-mon[22747]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:29:31.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:29:31 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:29:31] "GET /metrics HTTP/1.1" 200 37990 "" "Prometheus/2.33.4" 2024-04-22T15:29:32.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:32 smithi145 ceph-mon[21899]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 951 B/s rd, 0 op/s 2024-04-22T15:29:32.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:32 smithi145 ceph-mon[21899]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:29:32.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:32 smithi145 ceph-mon[26606]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 951 B/s rd, 0 op/s 2024-04-22T15:29:32.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:32 smithi145 ceph-mon[26606]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:29:32.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:32 smithi172 ceph-mon[22747]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 951 B/s rd, 0 op/s 2024-04-22T15:29:32.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:32 smithi172 ceph-mon[22747]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:29:34.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:34 smithi145 ceph-mon[21899]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 584 B/s rd, 0 op/s 2024-04-22T15:29:34.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:34 smithi145 ceph-mon[26606]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 584 B/s rd, 0 op/s 2024-04-22T15:29:34.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:34 smithi172 ceph-mon[22747]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 584 B/s rd, 0 op/s 2024-04-22T15:29:36.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:36 smithi145 ceph-mon[21899]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:29:36.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:36 smithi145 ceph-mon[26606]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:29:36.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:36 smithi172 ceph-mon[22747]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:29:38.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:38 smithi145 ceph-mon[21899]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:29:38.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:38 smithi145 ceph-mon[26606]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:29:38.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:38 smithi172 ceph-mon[22747]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:29:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:39 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:29:39.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:39 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:29:39.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:39 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:29:40.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:40 smithi145 ceph-mon[26606]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:29:40.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:40 smithi145 ceph-mon[21899]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:29:40.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:40 smithi172 ceph-mon[22747]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:29:41.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:29:41 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:29:41] "GET /metrics HTTP/1.1" 200 37990 "" "Prometheus/2.33.4" 2024-04-22T15:29:42.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:42 smithi145 ceph-mon[21899]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:29:42.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:42 smithi145 ceph-mon[26606]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:29:42.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:42 smithi172 ceph-mon[22747]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:29:43.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:43 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:29:43.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:43 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:29:43.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:43 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:29:44.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:44 smithi145 ceph-mon[21899]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:44.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:44 smithi145 ceph-mon[26606]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:44.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:44 smithi172 ceph-mon[22747]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:46.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:46 smithi145 ceph-mon[21899]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:29:46.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:46 smithi145 ceph-mon[26606]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:29:46.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:46 smithi172 ceph-mon[22747]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:29:48.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:48 smithi145 ceph-mon[21899]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:48.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:48 smithi145 ceph-mon[26606]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:48.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:48 smithi172 ceph-mon[22747]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:49.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:49 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:29:49.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:49 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:29:49.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:49 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:29:50.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:50 smithi145 ceph-mon[21899]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:29:50.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:50 smithi145 ceph-mon[26606]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:29:50.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:50 smithi172 ceph-mon[22747]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:29:51.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:29:51 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:29:51] "GET /metrics HTTP/1.1" 200 37990 "" "Prometheus/2.33.4" 2024-04-22T15:29:52.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:52 smithi145 ceph-mon[21899]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:52.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:52 smithi145 ceph-mon[26606]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:52.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:52 smithi172 ceph-mon[22747]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:54.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:54 smithi145 ceph-mon[21899]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:54.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:54 smithi145 ceph-mon[26606]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:54.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:54 smithi172 ceph-mon[22747]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:56.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:56 smithi145 ceph-mon[21899]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:29:56.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:56 smithi145 ceph-mon[26606]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:29:56.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:56 smithi172 ceph-mon[22747]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:29:58.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:58 smithi145 ceph-mon[21899]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:58.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:58 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:29:58.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:58 smithi145 ceph-mon[26606]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:58.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:58 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:29:58.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:58 smithi172 ceph-mon[22747]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:29:58.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:58 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:29:59.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:29:59 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:29:59.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:29:59 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:29:59.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:29:59 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:30:00.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:00 smithi172 ceph-mon[22747]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:00.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:00 smithi172 ceph-mon[22747]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in error state'] 2024-04-22T15:30:00.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:00 smithi172 ceph-mon[22747]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in error state'] 2024-04-22T15:30:00.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:00 smithi172 ceph-mon[22747]: daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in error state 2024-04-22T15:30:00.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:00 smithi145 ceph-mon[21899]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:00.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:00 smithi145 ceph-mon[21899]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in error state'] 2024-04-22T15:30:00.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:00 smithi145 ceph-mon[21899]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in error state'] 2024-04-22T15:30:00.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:00 smithi145 ceph-mon[21899]: daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in error state 2024-04-22T15:30:00.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:00 smithi145 ceph-mon[26606]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:00.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:00 smithi145 ceph-mon[26606]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in error state'] 2024-04-22T15:30:00.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:00 smithi145 ceph-mon[26606]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in error state'] 2024-04-22T15:30:00.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:00 smithi145 ceph-mon[26606]: daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in error state 2024-04-22T15:30:01.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:30:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:30:01] "GET /metrics HTTP/1.1" 200 37988 "" "Prometheus/2.33.4" 2024-04-22T15:30:02.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:02 smithi172 ceph-mon[22747]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:02.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:02 smithi145 ceph-mon[21899]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:02.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:02 smithi145 ceph-mon[26606]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:04.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:04 smithi172 ceph-mon[22747]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:04.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:04 smithi145 ceph-mon[21899]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:04.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:04 smithi145 ceph-mon[26606]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:06.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:06 smithi172 ceph-mon[22747]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:06.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:06 smithi145 ceph-mon[21899]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:06.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:06 smithi145 ceph-mon[26606]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:08.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:08 smithi172 ceph-mon[22747]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:08.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:08 smithi145 ceph-mon[21899]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:08.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:08 smithi145 ceph-mon[26606]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:09.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:09 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:30:09.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:09 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:30:09.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:09 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:30:10.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:10 smithi172 ceph-mon[22747]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:10.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:10 smithi145 ceph-mon[21899]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:10.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:10 smithi145 ceph-mon[26606]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:11.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:30:11 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:30:11] "GET /metrics HTTP/1.1" 200 37988 "" "Prometheus/2.33.4" 2024-04-22T15:30:12.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:12 smithi172 ceph-mon[22747]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:12.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:12 smithi145 ceph-mon[21899]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:12.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:12 smithi145 ceph-mon[26606]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:13.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:13 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:30:13.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:13 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:30:13.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:13 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:30:14.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:14 smithi172 ceph-mon[22747]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:14.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:14 smithi145 ceph-mon[21899]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:14.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:14 smithi145 ceph-mon[26606]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:16.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:16 smithi172 ceph-mon[22747]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:16.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:16 smithi145 ceph-mon[21899]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:16.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:16 smithi145 ceph-mon[26606]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:18.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:18 smithi172 ceph-mon[22747]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:18.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:18 smithi145 ceph-mon[21899]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:18.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:18 smithi145 ceph-mon[26606]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:19.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:19 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:30:19.774 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:19 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:30:19.775 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:19 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:30:20.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:20 smithi172 ceph-mon[22747]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:20.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:20 smithi145 ceph-mon[21899]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:20.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:20 smithi145 ceph-mon[26606]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:21.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:30:21 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:30:21] "GET /metrics HTTP/1.1" 200 37990 "" "Prometheus/2.33.4" 2024-04-22T15:30:22.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:22 smithi172 ceph-mon[22747]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:22.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:22 smithi145 ceph-mon[21899]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:22.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:22 smithi145 ceph-mon[26606]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:24.718 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:24 smithi145 ceph-mon[21899]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:24.718 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:24 smithi145 ceph-mon[26606]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:24.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:24 smithi172 ceph-mon[22747]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:26.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:26 smithi172 ceph-mon[22747]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:26.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:26 smithi145 ceph-mon[21899]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:26.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:26 smithi145 ceph-mon[26606]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:28.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:28 smithi172 ceph-mon[22747]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:28.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:28 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:30:28.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:28 smithi145 ceph-mon[21899]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:28.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:28 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:30:28.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:28 smithi145 ceph-mon[26606]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:28.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:28 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:30:29.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:29 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:30:29.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:29 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:30:29.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:29 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:30:30.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:30 smithi172 ceph-mon[22747]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:30.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:30 smithi145 ceph-mon[21899]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:30.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:30 smithi145 ceph-mon[26606]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:31.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:31 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:30:31.718 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:30:31 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:30:31] "GET /metrics HTTP/1.1" 200 37994 "" "Prometheus/2.33.4" 2024-04-22T15:30:31.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:31 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:30:31.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:31 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:30:32.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:32 smithi172 ceph-mon[22747]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:32.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:32 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:30:32.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:32 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:30:32.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:32 smithi172 ceph-mon[22747]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:30:32.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:32 smithi145 ceph-mon[21899]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:32.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:32 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:30:32.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:32 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:30:32.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:32 smithi145 ceph-mon[21899]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:30:32.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:32 smithi145 ceph-mon[26606]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:32.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:32 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:30:32.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:32 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:30:32.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:32 smithi145 ceph-mon[26606]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:30:33.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:33 smithi172 ceph-mon[22747]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 958 B/s rd, 0 op/s 2024-04-22T15:30:33.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:33 smithi172 ceph-mon[22747]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:33.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:33 smithi145 ceph-mon[21899]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 958 B/s rd, 0 op/s 2024-04-22T15:30:33.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:33 smithi145 ceph-mon[21899]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:33.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:33 smithi145 ceph-mon[26606]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 958 B/s rd, 0 op/s 2024-04-22T15:30:33.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:33 smithi145 ceph-mon[26606]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:35.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:35 smithi145 ceph-mon[21899]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:35.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:35 smithi145 ceph-mon[26606]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:35.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:35 smithi172 ceph-mon[22747]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:37.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:37 smithi145 ceph-mon[21899]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:37 smithi145 ceph-mon[26606]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:37.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:37 smithi172 ceph-mon[22747]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:39 smithi145 ceph-mon[21899]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 589 B/s rd, 0 op/s 2024-04-22T15:30:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:39 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:30:39.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:39 smithi145 ceph-mon[26606]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 589 B/s rd, 0 op/s 2024-04-22T15:30:39.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:39 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:30:39.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:39 smithi172 ceph-mon[22747]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 589 B/s rd, 0 op/s 2024-04-22T15:30:39.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:39 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:30:41.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:41 smithi145 ceph-mon[21899]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:41.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:41 smithi145 ceph-mon[26606]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:41.687 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:41 smithi172 ceph-mon[22747]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:41.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:30:41 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:30:41] "GET /metrics HTTP/1.1" 200 37994 "" "Prometheus/2.33.4" 2024-04-22T15:30:43.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:43 smithi145 ceph-mon[26606]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:30:43.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:43 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:30:43.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:43 smithi145 ceph-mon[21899]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:30:43.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:43 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:30:43.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:43 smithi172 ceph-mon[22747]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:30:43.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:43 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:30:45.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:45 smithi145 ceph-mon[26606]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:45.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:45 smithi145 ceph-mon[21899]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:45.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:45 smithi172 ceph-mon[22747]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:47.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:47 smithi145 ceph-mon[26606]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:47.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:47 smithi145 ceph-mon[21899]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:47 smithi172 ceph-mon[22747]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:49.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:49 smithi145 ceph-mon[26606]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:49.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:49 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:30:49.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:49 smithi145 ceph-mon[21899]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:49.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:49 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:30:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:49 smithi172 ceph-mon[22747]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:49 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:30:51.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:51 smithi145 ceph-mon[21899]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:51.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:51 smithi145 ceph-mon[26606]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:51.687 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:51 smithi172 ceph-mon[22747]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:51.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:30:51 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:30:51] "GET /metrics HTTP/1.1" 200 37997 "" "Prometheus/2.33.4" 2024-04-22T15:30:53.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:53 smithi145 ceph-mon[21899]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:53.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:53 smithi145 ceph-mon[26606]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:53.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:53 smithi172 ceph-mon[22747]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:55.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:55 smithi145 ceph-mon[21899]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:55.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:55 smithi145 ceph-mon[26606]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:55.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:55 smithi172 ceph-mon[22747]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:30:57.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:57 smithi145 ceph-mon[26606]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:57.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:57 smithi145 ceph-mon[21899]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:57.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:57 smithi172 ceph-mon[22747]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:58.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:58 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:30:58.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:58 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:30:58.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:58 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:30:59.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:59 smithi145 ceph-mon[26606]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:59.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:30:59 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:30:59.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:59 smithi145 ceph-mon[21899]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:59.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:30:59 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:30:59.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:59 smithi172 ceph-mon[22747]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:30:59.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:30:59 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:31:01.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:01 smithi145 ceph-mon[26606]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:31:01.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:01 smithi145 ceph-mon[21899]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:31:01.687 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:01 smithi172 ceph-mon[22747]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:31:01.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:31:01] "GET /metrics HTTP/1.1" 200 37999 "" "Prometheus/2.33.4" 2024-04-22T15:31:03.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:03 smithi145 ceph-mon[26606]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:31:03.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:03 smithi145 ceph-mon[21899]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:31:03.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:03 smithi172 ceph-mon[22747]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:31:05.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:05 smithi145 ceph-mon[21899]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:31:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:05 smithi145 ceph-mon[26606]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:31:05.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:05 smithi172 ceph-mon[22747]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:31:07.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:07 smithi145 ceph-mon[21899]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:31:07.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:07 smithi145 ceph-mon[26606]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:31:07.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:07 smithi172 ceph-mon[22747]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:31:09.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:09 smithi145 ceph-mon[26606]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:31:09.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:09 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:31:09.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:09 smithi145 ceph-mon[21899]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:31:09.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:09 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:31:09.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:09 smithi172 ceph-mon[22747]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:31:09.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:09 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:31:11.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:11 smithi145 ceph-mon[21899]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:31:11.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:11 smithi145 ceph-mon[26606]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:31:11.687 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:11 smithi172 ceph-mon[22747]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:31:11.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:11 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:31:11] "GET /metrics HTTP/1.1" 200 37999 "" "Prometheus/2.33.4" 2024-04-22T15:31:13.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:13 smithi145 ceph-mon[26606]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:31:13.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:13 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:31:13.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:13 smithi145 ceph-mon[21899]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:31:13.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:13 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:31:13.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:13 smithi172 ceph-mon[22747]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:31:13.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:13 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:31:15.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:15 smithi145 ceph-mon[26606]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:31:15.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:15 smithi145 ceph-mon[21899]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:31:15.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:15 smithi172 ceph-mon[22747]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:31:17.119 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch ps' 2024-04-22T15:31:17.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:17 smithi145 ceph-mon[21899]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:31:17.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:17 smithi145 ceph-mon[26606]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:31:17.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:17 smithi172 ceph-mon[22747]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:31:18.964 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:31:18.964 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (12m) 3m ago 20m 25.1M - 0.23.0 ba2b418f427c 2b762f668fa7 2024-04-22T15:31:18.964 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (12m) 2m ago 20m 52.9M - 8.3.5 dad864ee21e9 20a9f6cd1859 2024-04-22T15:31:18.965 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 error 3m ago 19m 54.6M - 3.5 e1d6a67b021e 9704631ffa0e 2024-04-22T15:31:18.965 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443,9283 running (15m) 2m ago 25m 493M - 19.0.0-3223-g5883d494 3d983c26e278 1920e9785cbb 2024-04-22T15:31:18.965 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (9m) 3m ago 26m 74.0M - 19.0.0-3223-g5883d494 3d983c26e278 b2de97566cf4 2024-04-22T15:31:18.965 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (26m) 3m ago 26m 71.5M 2048M 17.2.0 e1d6a67b021e 28106a7bf16e 2024-04-22T15:31:18.965 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (25m) 2m ago 25m 54.2M 2048M 17.2.0 e1d6a67b021e a2fc3297953a 2024-04-22T15:31:18.965 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (25m) 3m ago 25m 48.7M 2048M 17.2.0 e1d6a67b021e b71d9b9f8509 2024-04-22T15:31:18.965 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (12m) 3m ago 21m 16.0M - 1.3.1 1dbe0e931976 7a7a9854f247 2024-04-22T15:31:18.965 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (12m) 2m ago 21m 18.6M - 1.3.1 1dbe0e931976 e5b0a512c835 2024-04-22T15:31:18.965 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (24m) 3m ago 24m 55.7M 2992M 17.2.0 e1d6a67b021e ea9cdc639916 2024-04-22T15:31:18.965 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (24m) 3m ago 24m 53.9M 2992M 17.2.0 e1d6a67b021e 3b9b37a8781f 2024-04-22T15:31:18.965 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (23m) 3m ago 23m 50.5M 2992M 17.2.0 e1d6a67b021e d9502dded8f8 2024-04-22T15:31:18.965 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (23m) 3m ago 23m 51.7M 2992M 17.2.0 e1d6a67b021e 8c72f30e54fc 2024-04-22T15:31:18.965 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (23m) 2m ago 23m 55.5M 3248M 17.2.0 e1d6a67b021e 80001f97ed35 2024-04-22T15:31:18.965 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (22m) 2m ago 22m 53.2M 3248M 17.2.0 e1d6a67b021e ec3ab1335b5c 2024-04-22T15:31:18.966 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (22m) 2m ago 22m 53.1M 3248M 17.2.0 e1d6a67b021e 45d475edf1ca 2024-04-22T15:31:18.966 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (22m) 2m ago 22m 52.6M 3248M 17.2.0 e1d6a67b021e cc9e1d9e7916 2024-04-22T15:31:18.966 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (2m) 2m ago 21m 42.2M - 2.33.4 514e6a882f6e 6d94243c82c3 2024-04-22T15:31:18.966 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (19m) 3m ago 19m 76.7M - 17.2.0 e1d6a67b021e c90a636c947e 2024-04-22T15:31:18.966 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (19m) 2m ago 19m 74.6M - 17.2.0 e1d6a67b021e 20ac46e9bfda 2024-04-22T15:31:19.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:19 smithi145 ceph-mon[21899]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:31:19.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:19 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:31:19.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:19 smithi145 ceph-mon[26606]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:31:19.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:19 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:31:19.677 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch ls' 2024-04-22T15:31:19.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:19 smithi172 ceph-mon[22747]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:31:19.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:19 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:31:20.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:20 smithi145 ceph-mon[21899]: from='client.15411 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:31:20.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:20 smithi145 ceph-mon[26606]: from='client.15411 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:31:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:20 smithi172 ceph-mon[22747]: from='client.15411 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:31:21.420 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:21 smithi145 ceph-mon[26606]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:31:21.420 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:21 smithi145 ceph-mon[21899]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:31:21.687 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:21 smithi172 ceph-mon[22747]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:31:21.738 INFO:teuthology.orchestra.run.smithi145.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-04-22T15:31:21.738 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager ?:9093,9094 1/1 3m ago 21m smithi145=a;count:1 2024-04-22T15:31:21.738 INFO:teuthology.orchestra.run.smithi145.stdout:grafana ?:3000 1/1 2m ago 21m smithi172=a;count:1 2024-04-22T15:31:21.738 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo ?:5000 0/1 3m ago 19m count:1 2024-04-22T15:31:21.738 INFO:teuthology.orchestra.run.smithi145.stdout:mgr 2/2 3m ago 25m smithi145=y;smithi172=x;count:2 2024-04-22T15:31:21.738 INFO:teuthology.orchestra.run.smithi145.stdout:mon 3/3 3m ago 25m smithi145:172.21.15.145=a;smithi145:[v2:172.21.15.145:3301,v1:172.21.15.145:6790]=c;smithi172:172.21.15.172=b;count:3 2024-04-22T15:31:21.738 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter ?:9100 2/2 3m ago 21m smithi145=a;smithi172=b;count:2 2024-04-22T15:31:21.738 INFO:teuthology.orchestra.run.smithi145.stdout:osd 8 3m ago - 2024-04-22T15:31:21.738 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus ?:9095 1/1 2m ago 21m smithi172=a;count:1 2024-04-22T15:31:21.738 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo ?:8000 2/2 3m ago 19m count:2 2024-04-22T15:31:21.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:21 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:31:21] "GET /metrics HTTP/1.1" 200 37998 "" "Prometheus/2.33.4" 2024-04-22T15:31:22.344 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph versions' 2024-04-22T15:31:23.594 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:23 smithi145 ceph-mon[26606]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:31:23.594 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:23 smithi145 ceph-mon[26606]: from='client.15417 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:31:23.595 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:23 smithi145 ceph-mon[21899]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:31:23.595 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:23 smithi145 ceph-mon[21899]: from='client.15417 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:31:23.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:23 smithi172 ceph-mon[22747]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:31:23.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:23 smithi172 ceph-mon[22747]: from='client.15417 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:31:24.388 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:31:24.388 INFO:teuthology.orchestra.run.smithi145.stdout: "mon": { 2024-04-22T15:31:24.388 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-22T15:31:24.388 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:31:24.388 INFO:teuthology.orchestra.run.smithi145.stdout: "mgr": { 2024-04-22T15:31:24.388 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:31:24.388 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:31:24.388 INFO:teuthology.orchestra.run.smithi145.stdout: "osd": { 2024-04-22T15:31:24.388 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T15:31:24.389 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:31:24.389 INFO:teuthology.orchestra.run.smithi145.stdout: "mds": {}, 2024-04-22T15:31:24.389 INFO:teuthology.orchestra.run.smithi145.stdout: "rgw": { 2024-04-22T15:31:24.389 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:31:24.389 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:31:24.389 INFO:teuthology.orchestra.run.smithi145.stdout: "overall": { 2024-04-22T15:31:24.389 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-22T15:31:24.389 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:31:24.389 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:31:24.389 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:31:25.000 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr' 2024-04-22T15:31:25.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:25 smithi145 ceph-mon[21899]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:31:25.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:25 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3417018214' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:31:25.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:25 smithi145 ceph-mon[26606]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:31:25.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:25 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3417018214' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:31:25.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:25 smithi172 ceph-mon[22747]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:31:25.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:25 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/3417018214' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:31:27.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:27 smithi172 ceph-mon[22747]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:31:27.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:27 smithi145 ceph-mon[21899]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:31:27.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:27 smithi145 ceph-mon[26606]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:31:28.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:28 smithi145 ceph-mon[21899]: from='client.15429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:31:28.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:28 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:31:28.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:28 smithi145 ceph-mon[26606]: from='client.15429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:31:28.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:28 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:31:28.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:28 smithi172 ceph-mon[22747]: from='client.15429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:31:28.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:28 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:31:29.580 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:29 smithi145 ceph-mon[26606]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:31:29.580 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:29 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:31:29.580 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:29 smithi145 ceph-mon[21899]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:31:29.580 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:29 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:31:29.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:29 smithi172 ceph-mon[22747]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:31:29.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:29 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:31:30.448 INFO:teuthology.orchestra.run.smithi145.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:31:31.072 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-22T15:31:31.664 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:31 smithi145 ceph-mon[26606]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:31:31.664 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:31 smithi145 ceph-mon[26606]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:31:31.665 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:31 smithi145 ceph-mon[26606]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:31:31.665 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:31 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:31:31.665 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:31 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:31:31.665 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:31 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:31:31.665 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:31 smithi145 ceph-mon[26606]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:31:31.665 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:31 smithi145 ceph-mon[21899]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:31:31.665 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:31 smithi145 ceph-mon[21899]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:31:31.665 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:31 smithi145 ceph-mon[21899]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:31:31.665 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:31 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:31:31.666 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:31 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:31:31.666 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:31 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:31:31.666 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:31 smithi145 ceph-mon[21899]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:31:31.718 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:31 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:31:31] "GET /metrics HTTP/1.1" 200 37996 "" "Prometheus/2.33.4" 2024-04-22T15:31:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:31 smithi172 ceph-mon[22747]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:31:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:31 smithi172 ceph-mon[22747]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:31:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:31 smithi172 ceph-mon[22747]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:31:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:31 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:31:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:31 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:31:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:31 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:31:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:31 smithi172 ceph-mon[22747]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:31:32.713 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:32 smithi145 ceph-mon[21899]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 941 B/s rd, 0 op/s 2024-04-22T15:31:32.713 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:32 smithi145 ceph-mon[21899]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:31:32.713 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:32 smithi145 ceph-mon[21899]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:31:32.713 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:32 smithi145 ceph-mon[26606]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 941 B/s rd, 0 op/s 2024-04-22T15:31:32.714 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:32 smithi145 ceph-mon[26606]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:31:32.714 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:32 smithi145 ceph-mon[26606]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:31:32.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:32 smithi172 ceph-mon[22747]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 941 B/s rd, 0 op/s 2024-04-22T15:31:32.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:32 smithi172 ceph-mon[22747]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:31:32.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:32 smithi172 ceph-mon[22747]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:31:34.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:34 smithi145 ceph-mon[21899]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 576 B/s rd, 0 op/s 2024-04-22T15:31:34.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:34 smithi145 ceph-mon[21899]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:31:34.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:34 smithi145 ceph-mon[21899]: Upgrade: Target is version 19.0.0-3223-g5883d494 (squid) 2024-04-22T15:31:34.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:34 smithi145 ceph-mon[21899]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, digests ['quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb'] 2024-04-22T15:31:34.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:34 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:31:34.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:34 smithi145 ceph-mon[21899]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-22T15:31:34.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:34 smithi145 ceph-mon[21899]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-22T15:31:34.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:34 smithi145 ceph-mon[21899]: Failing over to other MGR 2024-04-22T15:31:34.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:34 smithi145 ceph-mon[21899]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-22T15:31:34.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:34 smithi145 ceph-mon[21899]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-22T15:31:34.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:34 smithi145 ceph-mon[21899]: osdmap e107: 8 total, 8 up, 8 in 2024-04-22T15:31:34.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:34 smithi145 ceph-mon[21899]: Standby manager daemon x started 2024-04-22T15:31:34.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:34 smithi145 ceph-mon[26606]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 576 B/s rd, 0 op/s 2024-04-22T15:31:34.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:34 smithi145 ceph-mon[26606]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:31:34.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:34 smithi145 ceph-mon[26606]: Upgrade: Target is version 19.0.0-3223-g5883d494 (squid) 2024-04-22T15:31:34.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:34 smithi145 ceph-mon[26606]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, digests ['quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb'] 2024-04-22T15:31:34.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:34 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:31:34.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:34 smithi145 ceph-mon[26606]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-22T15:31:34.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:34 smithi145 ceph-mon[26606]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-22T15:31:34.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:34 smithi145 ceph-mon[26606]: Failing over to other MGR 2024-04-22T15:31:34.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:34 smithi145 ceph-mon[26606]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-22T15:31:34.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:34 smithi145 ceph-mon[26606]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-22T15:31:34.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:34 smithi145 ceph-mon[26606]: osdmap e107: 8 total, 8 up, 8 in 2024-04-22T15:31:34.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:34 smithi145 ceph-mon[26606]: Standby manager daemon x started 2024-04-22T15:31:34.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:34 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:31:34] ENGINE Bus STOPPING 2024-04-22T15:31:34.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:34 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:31:34] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-22T15:31:34.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:34 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:31:34] ENGINE Bus STOPPED 2024-04-22T15:31:34.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:34 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:31:34] ENGINE Bus STARTING 2024-04-22T15:31:34.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:34 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:31:34] ENGINE Serving on http://:::9283 2024-04-22T15:31:34.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:34 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:31:34] ENGINE Bus STARTED 2024-04-22T15:31:34.717 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:34 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:31:34.341+0000 7fb8db983700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-22T15:31:34.717 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:34 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ignoring --setuser ceph since I am not root 2024-04-22T15:31:34.717 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:34 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: ignoring --setgroup ceph since I am not root 2024-04-22T15:31:34.717 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:34 smithi172 ceph-mgr[42978]: -- 172.21.15.172:0/3099035752 <== mon.2 v2:172.21.15.172:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55dbd50b45a0 con 0x55dbd5088800 2024-04-22T15:31:34.717 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:34 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:31:34.523+0000 7f9384fd7200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T15:31:34.718 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:34 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:31:34.713+0000 7f9384fd7200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T15:31:34.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:34 smithi172 ceph-mon[22747]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail; 576 B/s rd, 0 op/s 2024-04-22T15:31:34.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:34 smithi172 ceph-mon[22747]: from='mgr.24956 ' entity='mgr.x' 2024-04-22T15:31:34.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:34 smithi172 ceph-mon[22747]: Upgrade: Target is version 19.0.0-3223-g5883d494 (squid) 2024-04-22T15:31:34.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:34 smithi172 ceph-mon[22747]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, digests ['quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb'] 2024-04-22T15:31:34.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:34 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:31:34.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:34 smithi172 ceph-mon[22747]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-22T15:31:34.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:34 smithi172 ceph-mon[22747]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-22T15:31:34.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:34 smithi172 ceph-mon[22747]: Failing over to other MGR 2024-04-22T15:31:34.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:34 smithi172 ceph-mon[22747]: from='mgr.24956 172.21.15.172:0/108401303' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-22T15:31:34.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:34 smithi172 ceph-mon[22747]: from='mgr.24956 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-22T15:31:34.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:34 smithi172 ceph-mon[22747]: osdmap e107: 8 total, 8 up, 8 in 2024-04-22T15:31:34.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:34 smithi172 ceph-mon[22747]: Standby manager daemon x started 2024-04-22T15:31:34.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:34 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:31:34.823+0000 7f9384fd7200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T15:31:35.382 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:31:35.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:35 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:31:35.157+0000 7f9384fd7200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T15:31:35.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:35 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:31:35.258+0000 7f9384fd7200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T15:31:35.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:35 smithi172 ceph-mon[22747]: from='mgr.24956 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-22T15:31:35.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:35 smithi172 ceph-mon[22747]: mgrmap e38: y(active, starting, since 0.997682s), standbys: x 2024-04-22T15:31:35.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:35 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:31:35.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:35 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:31:35.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:35 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:31:35.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:35 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T15:31:35.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:35 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T15:31:35.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:35 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:31:35.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:35 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:31:35.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:35 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:31:35.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:35 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:31:35.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:35 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:31:35.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:35 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:31:35.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:35 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:31:35.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:35 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:31:35.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:35 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:31:35.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:35 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:31:35.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:35 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:31:35.470 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:35 smithi172 ceph-mon[22747]: Manager daemon y is now available 2024-04-22T15:31:35.470 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:35 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:31:35.470 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:35 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:31:35.470 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:35 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:31:35.470 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:35 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:31:35.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[21899]: from='mgr.24956 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-22T15:31:35.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[21899]: mgrmap e38: y(active, starting, since 0.997682s), standbys: x 2024-04-22T15:31:35.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:31:35.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:31:35.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:31:35.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T15:31:35.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T15:31:35.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:31:35.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:31:35.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:31:35.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:31:35.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:31:35.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:31:35.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:31:35.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:31:35.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:31:35.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:31:35.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:31:35.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[21899]: Manager daemon y is now available 2024-04-22T15:31:35.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:31:35.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:31:35.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:31:35.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:31:35.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[26606]: from='mgr.24956 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-22T15:31:35.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[26606]: mgrmap e38: y(active, starting, since 0.997682s), standbys: x 2024-04-22T15:31:35.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:31:35.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:31:35.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:31:35.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T15:31:35.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T15:31:35.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:31:35.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:31:35.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:31:35.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:31:35.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:31:35.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:31:35.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:31:35.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:31:35.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:31:35.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:31:35.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:31:35.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[26606]: Manager daemon y is now available 2024-04-22T15:31:35.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:31:35.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:31:35.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:31:35.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:35 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:31:36.055 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:31:36.055 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (12m) 3m ago 20m 25.1M - 0.23.0 ba2b418f427c 2b762f668fa7 2024-04-22T15:31:36.056 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (12m) 3m ago 20m 52.9M - 8.3.5 dad864ee21e9 20a9f6cd1859 2024-04-22T15:31:36.056 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 error 3m ago 19m 54.6M - 3.5 e1d6a67b021e 9704631ffa0e 2024-04-22T15:31:36.056 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443,9283 running (15m) 3m ago 25m 493M - 19.0.0-3223-g5883d494 3d983c26e278 1920e9785cbb 2024-04-22T15:31:36.056 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (9m) 3m ago 26m 74.0M - 19.0.0-3223-g5883d494 3d983c26e278 b2de97566cf4 2024-04-22T15:31:36.056 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (26m) 3m ago 27m 71.5M 2048M 17.2.0 e1d6a67b021e 28106a7bf16e 2024-04-22T15:31:36.056 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (25m) 3m ago 25m 54.2M 2048M 17.2.0 e1d6a67b021e a2fc3297953a 2024-04-22T15:31:36.056 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (25m) 3m ago 25m 48.7M 2048M 17.2.0 e1d6a67b021e b71d9b9f8509 2024-04-22T15:31:36.056 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (12m) 3m ago 21m 16.0M - 1.3.1 1dbe0e931976 7a7a9854f247 2024-04-22T15:31:36.056 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (12m) 3m ago 21m 18.6M - 1.3.1 1dbe0e931976 e5b0a512c835 2024-04-22T15:31:36.056 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (25m) 3m ago 25m 55.7M 2992M 17.2.0 e1d6a67b021e ea9cdc639916 2024-04-22T15:31:36.056 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (24m) 3m ago 24m 53.9M 2992M 17.2.0 e1d6a67b021e 3b9b37a8781f 2024-04-22T15:31:36.056 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (24m) 3m ago 24m 50.5M 2992M 17.2.0 e1d6a67b021e d9502dded8f8 2024-04-22T15:31:36.056 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (23m) 3m ago 23m 51.7M 2992M 17.2.0 e1d6a67b021e 8c72f30e54fc 2024-04-22T15:31:36.056 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (23m) 3m ago 23m 55.5M 3248M 17.2.0 e1d6a67b021e 80001f97ed35 2024-04-22T15:31:36.057 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (23m) 3m ago 23m 53.2M 3248M 17.2.0 e1d6a67b021e ec3ab1335b5c 2024-04-22T15:31:36.057 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (22m) 3m ago 22m 53.1M 3248M 17.2.0 e1d6a67b021e 45d475edf1ca 2024-04-22T15:31:36.057 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (22m) 3m ago 22m 52.6M 3248M 17.2.0 e1d6a67b021e cc9e1d9e7916 2024-04-22T15:31:36.057 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (3m) 3m ago 21m 42.2M - 2.33.4 514e6a882f6e 6d94243c82c3 2024-04-22T15:31:36.057 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (19m) 3m ago 19m 76.7M - 17.2.0 e1d6a67b021e c90a636c947e 2024-04-22T15:31:36.057 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (19m) 3m ago 19m 74.6M - 17.2.0 e1d6a67b021e 20ac46e9bfda 2024-04-22T15:31:36.491 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:31:36.492 INFO:teuthology.orchestra.run.smithi145.stdout: "mon": { 2024-04-22T15:31:36.492 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-22T15:31:36.492 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:31:36.492 INFO:teuthology.orchestra.run.smithi145.stdout: "mgr": { 2024-04-22T15:31:36.492 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:31:36.492 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:31:36.492 INFO:teuthology.orchestra.run.smithi145.stdout: "osd": { 2024-04-22T15:31:36.492 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T15:31:36.492 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:31:36.492 INFO:teuthology.orchestra.run.smithi145.stdout: "mds": {}, 2024-04-22T15:31:36.492 INFO:teuthology.orchestra.run.smithi145.stdout: "rgw": { 2024-04-22T15:31:36.492 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:31:36.492 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:31:36.493 INFO:teuthology.orchestra.run.smithi145.stdout: "overall": { 2024-04-22T15:31:36.493 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-22T15:31:36.493 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:31:36.493 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:31:36.493 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:31:36.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:36 smithi145 ceph-mon[21899]: [22/Apr/2024:15:31:34] ENGINE Bus STARTING 2024-04-22T15:31:36.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:36 smithi145 ceph-mon[21899]: [22/Apr/2024:15:31:34] ENGINE Serving on https://172.21.15.145:7150 2024-04-22T15:31:36.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:36 smithi145 ceph-mon[21899]: [22/Apr/2024:15:31:34] ENGINE Serving on http://172.21.15.145:8765 2024-04-22T15:31:36.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:36 smithi145 ceph-mon[21899]: [22/Apr/2024:15:31:34] ENGINE Bus STARTED 2024-04-22T15:31:36.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:36 smithi145 ceph-mon[21899]: mgrmap e39: y(active, since 2s), standbys: x 2024-04-22T15:31:36.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:36 smithi145 ceph-mon[21899]: from='client.25354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:31:36.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:36 smithi145 ceph-mon[21899]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:31:36.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:36 smithi145 ceph-mon[26606]: [22/Apr/2024:15:31:34] ENGINE Bus STARTING 2024-04-22T15:31:36.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:36 smithi145 ceph-mon[26606]: [22/Apr/2024:15:31:34] ENGINE Serving on https://172.21.15.145:7150 2024-04-22T15:31:36.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:36 smithi145 ceph-mon[26606]: [22/Apr/2024:15:31:34] ENGINE Serving on http://172.21.15.145:8765 2024-04-22T15:31:36.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:36 smithi145 ceph-mon[26606]: [22/Apr/2024:15:31:34] ENGINE Bus STARTED 2024-04-22T15:31:36.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:36 smithi145 ceph-mon[26606]: mgrmap e39: y(active, since 2s), standbys: x 2024-04-22T15:31:36.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:36 smithi145 ceph-mon[26606]: from='client.25354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:31:36.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:36 smithi145 ceph-mon[26606]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:31:36.714 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:36 smithi172 ceph-mon[22747]: [22/Apr/2024:15:31:34] ENGINE Bus STARTING 2024-04-22T15:31:36.714 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:36 smithi172 ceph-mon[22747]: [22/Apr/2024:15:31:34] ENGINE Serving on https://172.21.15.145:7150 2024-04-22T15:31:36.714 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:36 smithi172 ceph-mon[22747]: [22/Apr/2024:15:31:34] ENGINE Serving on http://172.21.15.145:8765 2024-04-22T15:31:36.714 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:36 smithi172 ceph-mon[22747]: [22/Apr/2024:15:31:34] ENGINE Bus STARTED 2024-04-22T15:31:36.714 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:36 smithi172 ceph-mon[22747]: mgrmap e39: y(active, since 2s), standbys: x 2024-04-22T15:31:36.714 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:36 smithi172 ceph-mon[22747]: from='client.25354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:31:36.714 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:36 smithi172 ceph-mon[22747]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:31:36.715 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:36 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:31:36.530+0000 7f9384fd7200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T15:31:36.830 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:31:36.831 INFO:teuthology.orchestra.run.smithi145.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:31:36.831 INFO:teuthology.orchestra.run.smithi145.stdout: "in_progress": true, 2024-04-22T15:31:36.831 INFO:teuthology.orchestra.run.smithi145.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-04-22T15:31:36.831 INFO:teuthology.orchestra.run.smithi145.stdout: "services_complete": [ 2024-04-22T15:31:36.831 INFO:teuthology.orchestra.run.smithi145.stdout: "mgr" 2024-04-22T15:31:36.831 INFO:teuthology.orchestra.run.smithi145.stdout: ], 2024-04-22T15:31:36.831 INFO:teuthology.orchestra.run.smithi145.stdout: "progress": "2/2 daemons upgraded", 2024-04-22T15:31:36.831 INFO:teuthology.orchestra.run.smithi145.stdout: "message": "", 2024-04-22T15:31:36.831 INFO:teuthology.orchestra.run.smithi145.stdout: "is_paused": false 2024-04-22T15:31:36.831 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:31:36.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:36 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:31:36.712+0000 7f9384fd7200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T15:31:36.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:36 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:31:36.923+0000 7f9384fd7200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T15:31:37.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:37 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:31:37.258+0000 7f9384fd7200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T15:31:37.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:37 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:31:37.357+0000 7f9384fd7200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T15:31:37.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:37 smithi172 ceph-mon[22747]: from='client.15462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:31:37.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:37 smithi172 ceph-mon[22747]: from='client.15468 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:31:37.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:37 smithi172 ceph-mon[22747]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:31:37.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:37 smithi172 ceph-mon[22747]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in error state']) 2024-04-22T15:31:37.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:37 smithi172 ceph-mon[22747]: Cluster is now healthy 2024-04-22T15:31:37.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:37 smithi172 ceph-mon[22747]: mgrmap e40: y(active, since 3s), standbys: x 2024-04-22T15:31:37.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:37 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2010602709' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:31:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:37 smithi145 ceph-mon[26606]: from='client.15462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:31:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:37 smithi145 ceph-mon[26606]: from='client.15468 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:31:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:37 smithi145 ceph-mon[26606]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:31:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:37 smithi145 ceph-mon[26606]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in error state']) 2024-04-22T15:31:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:37 smithi145 ceph-mon[26606]: Cluster is now healthy 2024-04-22T15:31:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:37 smithi145 ceph-mon[26606]: mgrmap e40: y(active, since 3s), standbys: x 2024-04-22T15:31:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:37 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2010602709' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:31:37.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:37 smithi145 ceph-mon[21899]: from='client.15462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:31:37.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:37 smithi145 ceph-mon[21899]: from='client.15468 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:31:37.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:37 smithi145 ceph-mon[21899]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:31:37.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:37 smithi145 ceph-mon[21899]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in error state']) 2024-04-22T15:31:37.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:37 smithi145 ceph-mon[21899]: Cluster is now healthy 2024-04-22T15:31:37.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:37 smithi145 ceph-mon[21899]: mgrmap e40: y(active, since 3s), standbys: x 2024-04-22T15:31:37.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:37 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2010602709' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:31:37.912 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:37 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:31:37.574+0000 7f9384fd7200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T15:31:37.912 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:37 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:31:37.797+0000 7f9384fd7200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T15:31:38.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:38 smithi172 ceph-mon[22747]: from='client.25408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:31:38.467 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:38 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:31:38.463+0000 7f9384fd7200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T15:31:38.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:38 smithi145 ceph-mon[21899]: from='client.25408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:31:38.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:38 smithi145 ceph-mon[26606]: from='client.25408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:31:39.218 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:38 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:31:38.906+0000 7f9384fd7200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T15:31:39.218 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:39 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:31:39.005+0000 7f9384fd7200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T15:31:39.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:39 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:31:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:39 smithi145 ceph-mon[21899]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:31:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:39 smithi145 ceph-mon[21899]: mgrmap e41: y(active, since 5s), standbys: x 2024-04-22T15:31:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:39 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:39 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:39 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:39 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:39.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:39 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:31:39.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:39 smithi145 ceph-mon[26606]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:31:39.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:39 smithi145 ceph-mon[26606]: mgrmap e41: y(active, since 5s), standbys: x 2024-04-22T15:31:39.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:39 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:39.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:39 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:39.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:39 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:39.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:39 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:39.718 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:39 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:31:39.233+0000 7f9384fd7200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-22T15:31:39.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:39 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:31:39.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:39 smithi172 ceph-mon[22747]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:31:39.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:39 smithi172 ceph-mon[22747]: mgrmap e41: y(active, since 5s), standbys: x 2024-04-22T15:31:39.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:39 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:39.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:39 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:39.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:39 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:39.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:39 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:40.186 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:39 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:31:39.824+0000 7f9384fd7200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T15:31:40.186 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:39 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:31:39.924+0000 7f9384fd7200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T15:31:40.186 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:40 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:31:40.182+0000 7f9384fd7200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T15:31:40.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:40 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:31:40.322+0000 7f9384fd7200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T15:31:40.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:40 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:31:40.422+0000 7f9384fd7200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T15:31:40.870 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:40 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:31:40.547+0000 7f9384fd7200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T15:31:40.870 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:40 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:31:40.867+0000 7f9384fd7200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T15:31:41.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:41 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:41.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:41 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:41.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:41 smithi145 ceph-mon[21899]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:31:41.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:41 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:41.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:41 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:41.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:41 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:31:41.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:41 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:31:41.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:41 smithi145 ceph-mon[21899]: Updating smithi145:/etc/ceph/ceph.conf 2024-04-22T15:31:41.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:41 smithi145 ceph-mon[21899]: Updating smithi172:/etc/ceph/ceph.conf 2024-04-22T15:31:41.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:41 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:41.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:41 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:41.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:41 smithi145 ceph-mon[26606]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:31:41.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:41 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:41.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:41 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:41.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:41 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:31:41.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:41 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:31:41.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:41 smithi145 ceph-mon[26606]: Updating smithi145:/etc/ceph/ceph.conf 2024-04-22T15:31:41.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:41 smithi145 ceph-mon[26606]: Updating smithi172:/etc/ceph/ceph.conf 2024-04-22T15:31:41.718 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:41 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:31:41.516+0000 7f9384fd7200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T15:31:41.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:41 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:41.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:41 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:41.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:41 smithi172 ceph-mon[22747]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 91 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:31:41.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:41 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:41.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:41 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:41.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:41 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:31:41.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:41 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:31:41.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:41 smithi172 ceph-mon[22747]: Updating smithi145:/etc/ceph/ceph.conf 2024-04-22T15:31:41.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:41 smithi172 ceph-mon[22747]: Updating smithi172:/etc/ceph/ceph.conf 2024-04-22T15:31:42.218 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:41 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:31:41.769+0000 7f9384fd7200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T15:31:42.218 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:41 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:31:41] ENGINE Bus STARTING 2024-04-22T15:31:42.218 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:41 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: CherryPy Checker: 2024-04-22T15:31:42.218 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:41 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: The Application mounted at '' has an empty config. 2024-04-22T15:31:42.218 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:41 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: 2024-04-22T15:31:42.218 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:41 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:31:41] ENGINE Serving on http://:::9283 2024-04-22T15:31:42.218 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:41 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[42974]: [22/Apr/2024:15:31:41] ENGINE Bus STARTED 2024-04-22T15:31:42.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[21899]: Updating smithi145:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:31:42.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[21899]: Updating smithi172:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:31:42.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[21899]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:31:42.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[21899]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:31:42.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[21899]: Updating smithi145:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:31:42.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[21899]: Updating smithi172:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:31:42.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:42.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:42.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:42.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:42.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[21899]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 107 KiB/s rd, 0 B/s wr, 139 op/s 2024-04-22T15:31:42.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:42.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[21899]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:31:42.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:31:42.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:31:42.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[21899]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:31:42.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[21899]: Standby manager daemon x restarted 2024-04-22T15:31:42.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[21899]: Standby manager daemon x started 2024-04-22T15:31:42.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.172:0/2950953410' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T15:31:42.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.172:0/2950953410' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:31:42.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.172:0/2950953410' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T15:31:42.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.172:0/2950953410' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:31:42.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[26606]: Updating smithi145:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:31:42.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[26606]: Updating smithi172:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:31:42.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[26606]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:31:42.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[26606]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:31:42.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[26606]: Updating smithi145:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:31:42.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[26606]: Updating smithi172:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:31:42.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:42.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:42.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:42.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:42.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[26606]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 107 KiB/s rd, 0 B/s wr, 139 op/s 2024-04-22T15:31:42.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:42.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[26606]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:31:42.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:31:42.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:31:42.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[26606]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:31:42.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[26606]: Standby manager daemon x restarted 2024-04-22T15:31:42.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[26606]: Standby manager daemon x started 2024-04-22T15:31:42.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.172:0/2950953410' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T15:31:42.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.172:0/2950953410' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:31:42.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.172:0/2950953410' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T15:31:42.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:42 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.172:0/2950953410' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:31:42.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:42 smithi172 ceph-mon[22747]: Updating smithi145:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:31:42.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:42 smithi172 ceph-mon[22747]: Updating smithi172:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:31:42.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:42 smithi172 ceph-mon[22747]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:31:42.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:42 smithi172 ceph-mon[22747]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:31:42.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:42 smithi172 ceph-mon[22747]: Updating smithi145:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:31:42.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:42 smithi172 ceph-mon[22747]: Updating smithi172:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:31:42.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:42 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:42.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:42 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:42.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:42 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:42.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:42 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:42.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:42 smithi172 ceph-mon[22747]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 107 KiB/s rd, 0 B/s wr, 139 op/s 2024-04-22T15:31:42.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:42 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:42.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:42 smithi172 ceph-mon[22747]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:31:42.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:42 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:31:42.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:42 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:31:42.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:42 smithi172 ceph-mon[22747]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:31:42.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:42 smithi172 ceph-mon[22747]: Standby manager daemon x restarted 2024-04-22T15:31:42.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:42 smithi172 ceph-mon[22747]: Standby manager daemon x started 2024-04-22T15:31:42.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:42 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.172:0/2950953410' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T15:31:42.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:42 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.172:0/2950953410' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:31:42.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:42 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.172:0/2950953410' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T15:31:42.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:42 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.172:0/2950953410' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:31:43.298 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:31:43.101+0000 7f42a94bd700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:31:43.298 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:31:43.298 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:31:43.298 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:31:43.298 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:31:43.298 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:31:43.298 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:31:43.299 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:31:43.299 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:31:43.299 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:31:43.299 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:31:43.299 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:31:43.299 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:31:43.299 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:31:43.299 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:31:43.299 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:31:43.299 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:31:43.299 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:31:43.299 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:31:43.299 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:31:43.299 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:31:43.299 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:31:43.300 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:31:43.300 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:31:43.300 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:31:43.300 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:31:43.300 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:31:43.300 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:31:43.300 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:31:43.300 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:31:43.300 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:31:43.301 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:31:43.301 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:31:43.301 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:31:43.301 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:31:43.301 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:31:43.301 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:31:43.301 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:31:43.301 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:31:43.301 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:31:43.302 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:31:43.302 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:31:43.302 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:31:43.302 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:31:43.302 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:31:43.302 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:31:43.302 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:31:43.302 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:31:43.303 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:31:43.303 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:31:43.303 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:31:43.303 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:31:43.303 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:31:43.303 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:31:43.303 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:31:43.303 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:31:43.303 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:31:43.303 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:31:43.303 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:31:43.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-mon[21899]: Health check failed: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in error state'] (CEPHADM_FAILED_DAEMON) 2024-04-22T15:31:43.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-mon[21899]: mgrmap e42: y(active, since 9s), standbys: x 2024-04-22T15:31:43.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-mon[26606]: Health check failed: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in error state'] (CEPHADM_FAILED_DAEMON) 2024-04-22T15:31:43.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:43 smithi145 ceph-mon[26606]: mgrmap e42: y(active, since 9s), standbys: x 2024-04-22T15:31:43.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:43 smithi172 ceph-mon[22747]: Health check failed: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in error state'] (CEPHADM_FAILED_DAEMON) 2024-04-22T15:31:43.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:43 smithi172 ceph-mon[22747]: mgrmap e42: y(active, since 9s), standbys: x 2024-04-22T15:31:44.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:44 smithi172 ceph-mon[22747]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:31:44.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:31:44.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:31:44.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:31:44.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:31:44.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:31:44.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:31:44.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:31:44.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:31:44.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:31:44.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:31:44.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:31:44.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:31:44.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:31:44.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:31:44.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:31:44.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:31:44.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:31:44.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:31:44.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:31:44.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:31:44.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:31:44.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:31:44.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:31:44.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:31:44.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:31:44.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:31:44.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:31:44.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:31:44.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:31:44.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:31:44.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:31:44.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:31:44.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:31:44.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:31:44.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:31:44.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:31:44.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:31:44.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:31:44.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:31:44.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:31:44.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:31:44.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:31:44.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:31:44.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:31:44.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:31:44.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:31:44.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:31:44.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:31:44.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:31:44.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:31:44.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:31:44.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:31:44.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:31:44.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:31:44.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:31:44.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:31:44.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:31:44.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:31:44.472 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:44 smithi172 ceph-mon[22747]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T15:31:44.472 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:44 smithi172 ceph-mon[22747]: Reconfiguring daemon prometheus.a on smithi172 2024-04-22T15:31:44.472 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:44 smithi172 ceph-mon[22747]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 0 B/s wr, 104 op/s 2024-04-22T15:31:44.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:44 smithi145 ceph-mon[21899]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:31:44.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:31:44.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:31:44.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:31:44.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:31:44.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:31:44.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:31:44.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:31:44.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:31:44.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:31:44.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:31:44.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:31:44.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:31:44.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:31:44.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:31:44.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:31:44.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:31:44.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:31:44.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:31:44.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:31:44.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:31:44.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:31:44.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:31:44.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:31:44.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:31:44.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:31:44.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:31:44.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:31:44.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:31:44.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:31:44.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:31:44.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:31:44.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:31:44.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:31:44.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:31:44.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:31:44.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:31:44.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:31:44.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:31:44.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:31:44.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:31:44.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:31:44.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:31:44.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:31:44.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:31:44.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:31:44.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:31:44.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:31:44.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:31:44.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:31:44.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:31:44.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:31:44.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:31:44.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:31:44.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:44 smithi145 ceph-mon[26606]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:31:44.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:31:44.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:31:44.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:31:44.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:31:44.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:31:44.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:31:44.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:31:44.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:31:44.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:31:44.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:31:44.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:31:44.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:31:44.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:31:44.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:31:44.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:31:44.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:31:44.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:31:44.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:31:44.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:31:44.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:31:44.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:31:44.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:31:44.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:31:44.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:31:44.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:31:44.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:31:44.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:31:44.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:31:44.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:31:44.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:31:44.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:31:44.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:31:44.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:31:44.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:31:44.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:31:44.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:31:44.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:31:44.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:31:44.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:31:44.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:31:44.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:31:44.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:31:44.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:31:44.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:31:44.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:31:44.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:31:44.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:31:44.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:31:44.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:31:44.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:31:44.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:31:44.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:31:44.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:31:44.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:31:44.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:31:44.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:31:44.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:31:44.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:31:44.631 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:44 smithi145 ceph-mon[21899]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T15:31:44.631 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:44 smithi145 ceph-mon[21899]: Reconfiguring daemon prometheus.a on smithi172 2024-04-22T15:31:44.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:44 smithi145 ceph-mon[21899]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 0 B/s wr, 104 op/s 2024-04-22T15:31:44.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:31:44.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:31:44.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:31:44.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:31:44.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:31:44.632 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:44 smithi145 ceph-mon[26606]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T15:31:44.632 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:44 smithi145 ceph-mon[26606]: Reconfiguring daemon prometheus.a on smithi172 2024-04-22T15:31:44.632 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:44 smithi145 ceph-mon[26606]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 0 B/s wr, 104 op/s 2024-04-22T15:31:46.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:46 smithi172 ceph-mon[22747]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s rd, 0 B/s wr, 83 op/s 2024-04-22T15:31:46.719 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:46 smithi172 systemd[1]: Stopping Ceph prometheus.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:31:46.719 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:46 smithi172 bash[57465]: Error: no container with name or ID "ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-22T15:31:46.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:46 smithi145 ceph-mon[21899]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s rd, 0 B/s wr, 83 op/s 2024-04-22T15:31:46.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:46 smithi145 ceph-mon[26606]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s rd, 0 B/s wr, 83 op/s 2024-04-22T15:31:46.998 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:46 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:31:46.735Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-22T15:31:46.998 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:46 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:31:46.736Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-22T15:31:46.998 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:46 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:31:46.736Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-22T15:31:46.998 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:46 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:31:46.736Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-22T15:31:46.999 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:46 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:31:46.736Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-22T15:31:46.999 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:46 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:31:46.736Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-22T15:31:46.999 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:46 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:31:46.736Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-22T15:31:46.999 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:46 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:31:46.736Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-22T15:31:46.999 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:46 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:31:46.736Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-22T15:31:46.999 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:46 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:31:46.736Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-22T15:31:46.999 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:46 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:31:46.736Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-22T15:31:46.999 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:46 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[54671]: ts=2024-04-22T15:31:46.736Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-22T15:31:46.999 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:46 smithi172 podman[57478]: 2024-04-22 15:31:46.746149107 +0000 UTC m=+0.156615182 container died 6d94243c82c3ccb9b4cd7166c007b278473eed869bf9143a8cdede7f4fa90d4f (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:31:47.364 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:47 smithi172 podman[57478]: 2024-04-22 15:31:47.094143996 +0000 UTC m=+0.504610082 container cleanup 6d94243c82c3ccb9b4cd7166c007b278473eed869bf9143a8cdede7f4fa90d4f (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:31:47.364 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:47 smithi172 bash[57478]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a 2024-04-22T15:31:47.364 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:47 smithi172 bash[57506]: Error: no container with name or ID "ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-22T15:31:47.694 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:47 smithi172 podman[57492]: 2024-04-22 15:31:47.361653114 +0000 UTC m=+0.616807961 container remove 6d94243c82c3ccb9b4cd7166c007b278473eed869bf9143a8cdede7f4fa90d4f (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:31:47.694 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:47 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-22T15:31:47.694 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:47 smithi172 systemd[1]: Stopped Ceph prometheus.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:31:47.694 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:47 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.435s CPU time. 2024-04-22T15:31:47.694 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:47 smithi172 systemd[1]: Starting Ceph prometheus.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:31:47.694 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:47 smithi172 podman[57576]: 2024-04-22 15:31:47.497254268 +0000 UTC m=+0.019250785 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-22T15:31:47.695 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:47 smithi172 podman[57576]: 2024-04-22 15:31:47.595969687 +0000 UTC m=+0.117966192 container create 83fc61bca2388aead7c3af6214ec4cef1bf55b417be7ddeec9a5fe3a3f6f919f (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:31:47.968 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:47 smithi172 podman[57576]: 2024-04-22 15:31:47.814754648 +0000 UTC m=+0.336751153 container init 83fc61bca2388aead7c3af6214ec4cef1bf55b417be7ddeec9a5fe3a3f6f919f (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:31:47.968 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:47 smithi172 podman[57576]: 2024-04-22 15:31:47.817439434 +0000 UTC m=+0.339435937 container start 83fc61bca2388aead7c3af6214ec4cef1bf55b417be7ddeec9a5fe3a3f6f919f (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:31:47.968 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:31:47.840Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-22T15:31:47.968 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:31:47.840Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-22T15:31:47.968 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:31:47.840Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-22T15:31:47.969 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:31:47.841Z caller=main.go:518 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi172 (none))" 2024-04-22T15:31:47.969 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:31:47.841Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-22T15:31:47.969 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:31:47.841Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-22T15:31:47.969 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:31:47.843Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-22T15:31:47.969 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:31:47.843Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-22T15:31:47.969 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:31:47.844Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-22T15:31:47.969 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:31:47.845Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-22T15:31:47.969 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:31:47.845Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.557µs 2024-04-22T15:31:47.969 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:31:47.845Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-22T15:31:47.969 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:47 smithi172 bash[57576]: 83fc61bca2388aead7c3af6214ec4cef1bf55b417be7ddeec9a5fe3a3f6f919f 2024-04-22T15:31:47.969 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:47 smithi172 systemd[1]: Started Ceph prometheus.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:31:48.330 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:31:48.059Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=5 2024-04-22T15:31:48.599 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:48 smithi172 ceph-mon[22747]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 0 B/s wr, 141 op/s 2024-04-22T15:31:48.599 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:48 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:48.599 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:48 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:48.599 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:48 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:31:48.599 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:48 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:31:48.599 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:31:48.327Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=5 2024-04-22T15:31:48.599 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:31:48.540Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=5 2024-04-22T15:31:48.600 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:31:48.543Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=5 2024-04-22T15:31:48.600 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:31:48.550Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=5 2024-04-22T15:31:48.600 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:31:48.550Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=5 2024-04-22T15:31:48.600 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:31:48.550Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=41.451µs wal_replay_duration=705.126336ms total_replay_duration=705.180498ms 2024-04-22T15:31:48.600 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:31:48.554Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-22T15:31:48.600 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:31:48.555Z caller=main.go:947 level=info msg="TSDB started" 2024-04-22T15:31:48.600 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:31:48.555Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-22T15:31:48.600 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:31:48.575Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=20.380498ms db_storage=714ns remote_storage=1.523µs web_handler=308ns query_engine=877ns scrape=236.103µs scrape_sd=93.929µs notify=16.038µs notify_sd=18.502µs rules=19.623753ms 2024-04-22T15:31:48.600 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:31:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:31:48.575Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-22T15:31:48.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:48 smithi145 ceph-mon[21899]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 0 B/s wr, 141 op/s 2024-04-22T15:31:48.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:48 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:48.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:48 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:48.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:48 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:31:48.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:48 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:31:48.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:48 smithi145 ceph-mon[26606]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 0 B/s wr, 141 op/s 2024-04-22T15:31:48.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:48 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:48.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:48 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:48.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:48 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:31:48.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:48 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:31:49.557 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:49 smithi172 ceph-mon[22747]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:31:49.557 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:49 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:31:49.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:49 smithi145 ceph-mon[26606]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:31:49.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:49 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:31:49.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:49 smithi145 ceph-mon[21899]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:31:49.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:49 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:31:50.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:50 smithi172 ceph-mon[22747]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 0 B/s wr, 141 op/s 2024-04-22T15:31:50.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:50 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:31:50.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:50 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:31:50.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:50 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:31:50.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:50 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:31:50.769 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:50 smithi145 ceph-mon[21899]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 0 B/s wr, 141 op/s 2024-04-22T15:31:50.769 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:50 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:31:50.769 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:50 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:31:50.769 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:50 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:31:50.769 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:50 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:31:50.770 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:50 smithi145 ceph-mon[26606]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 0 B/s wr, 141 op/s 2024-04-22T15:31:50.770 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:50 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:31:50.770 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:50 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:31:50.770 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:50 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:31:50.770 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:50 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:31:51.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:51 smithi172 systemd[1]: Stopping Ceph mgr.x for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:31:51.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:51 smithi172 ceph-mon[22747]: Upgrade: Updating mgr.x 2024-04-22T15:31:51.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:51 smithi172 ceph-mon[22747]: Deploying daemon mgr.x on smithi172 2024-04-22T15:31:51.676 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:51 smithi145 ceph-mon[26606]: Upgrade: Updating mgr.x 2024-04-22T15:31:51.677 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:51 smithi145 ceph-mon[26606]: Deploying daemon mgr.x on smithi172 2024-04-22T15:31:51.677 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:51 smithi145 ceph-mon[21899]: Upgrade: Updating mgr.x 2024-04-22T15:31:51.677 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:51 smithi145 ceph-mon[21899]: Deploying daemon mgr.x on smithi172 2024-04-22T15:31:51.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:51 smithi172 podman[57932]: 2024-04-22 15:31:51.663450497 +0000 UTC m=+0.126329490 container died 1920e9785cbb92f36c806b31a4ea16b6bcfe13b505c10dfd196bc4996397957c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-04-22T15:31:52.294 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:52 smithi172 podman[57932]: 2024-04-22 15:31:52.048925994 +0000 UTC m=+0.511805059 container cleanup 1920e9785cbb92f36c806b31a4ea16b6bcfe13b505c10dfd196bc4996397957c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-04-22T15:31:52.294 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:52 smithi172 bash[57932]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x 2024-04-22T15:31:52.613 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:52 smithi172 ceph-mon[22747]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 0 B/s wr, 141 op/s 2024-04-22T15:31:52.613 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:52 smithi172 podman[57946]: 2024-04-22 15:31:52.291639393 +0000 UTC m=+0.630830024 container remove 1920e9785cbb92f36c806b31a4ea16b6bcfe13b505c10dfd196bc4996397957c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-04-22T15:31:52.613 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:52 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-22T15:31:52.613 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:52 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-04-22T15:31:52.613 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:52 smithi172 systemd[1]: Stopped Ceph mgr.x for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:31:52.613 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:52 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mgr.x.service: Consumed 58.333s CPU time. 2024-04-22T15:31:52.871 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:52 smithi172 systemd[1]: Starting Ceph mgr.x for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:31:52.871 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:52 smithi172 podman[58079]: 2024-04-22 15:31:52.730738069 +0000 UTC m=+0.019064296 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:31:52.871 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:52 smithi172 podman[58079]: 2024-04-22 15:31:52.868356843 +0000 UTC m=+0.156683048 container create e7500ef71604b12a58c6d95ac7ea1e4da259e385e11bc16ba0f6f0f44339ac00 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-04-22T15:31:52.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:52 smithi145 ceph-mon[21899]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 0 B/s wr, 141 op/s 2024-04-22T15:31:52.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:52 smithi145 ceph-mon[26606]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 0 B/s wr, 141 op/s 2024-04-22T15:31:53.140 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:53 smithi172 podman[58079]: 2024-04-22 15:31:53.070784916 +0000 UTC m=+0.359111126 container init e7500ef71604b12a58c6d95ac7ea1e4da259e385e11bc16ba0f6f0f44339ac00 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T15:31:53.140 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:53 smithi172 podman[58079]: 2024-04-22 15:31:53.073298495 +0000 UTC m=+0.361624707 container start e7500ef71604b12a58c6d95ac7ea1e4da259e385e11bc16ba0f6f0f44339ac00 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T15:31:53.140 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:53 smithi172 ceph-mgr[58098]: -- 172.21.15.172:0/1826643417 <== mon.2 v2:172.21.15.172:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x564b0a2aa5a0 con 0x564b0a27ec00 2024-04-22T15:31:53.428 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:53 smithi172 bash[58079]: e7500ef71604b12a58c6d95ac7ea1e4da259e385e11bc16ba0f6f0f44339ac00 2024-04-22T15:31:53.429 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:53 smithi172 systemd[1]: Started Ceph mgr.x for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:31:53.429 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:31:53.229+0000 7fd5bc14f200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T15:31:53.429 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:31:53.425+0000 7fd5bc14f200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T15:31:53.718 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:31:53.536+0000 7fd5bc14f200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T15:31:54.210 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:31:53.880+0000 7fd5bc14f200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T15:31:54.210 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:31:53.965+0000 7fd5bc14f200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T15:31:54.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:54 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:54.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:54 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:54.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:54 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:31:54.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:54 smithi172 ceph-mon[22747]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-04-22T15:31:54.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:54 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:54.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:54 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:54.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:54 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:31:54.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:54 smithi145 ceph-mon[26606]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-04-22T15:31:54.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:54 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:54.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:54 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:54.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:54 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:31:54.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:54 smithi145 ceph-mon[21899]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-04-22T15:31:55.567 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:55 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:31:55.248+0000 7fd5bc14f200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T15:31:55.567 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:55 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:31:55.439+0000 7fd5bc14f200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T15:31:55.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:31:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:31:55] "GET /metrics HTTP/1.1" 200 37999 "" "Prometheus/2.33.4" 2024-04-22T15:31:55.917 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:55 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:31:55.619+0000 7fd5bc14f200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T15:31:55.917 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:55 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:31:55.913+0000 7fd5bc14f200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T15:31:56.217 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:56 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:31:56.012+0000 7fd5bc14f200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T15:31:56.571 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:56 smithi172 ceph-mon[22747]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-04-22T15:31:56.571 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:56 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:31:56.227+0000 7fd5bc14f200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T15:31:56.572 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:56 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:31:56.428+0000 7fd5bc14f200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T15:31:56.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:56 smithi145 ceph-mon[21899]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-04-22T15:31:56.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:56 smithi145 ceph-mon[26606]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-04-22T15:31:57.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:31:57.057+0000 7fd5bc14f200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T15:31:57.829 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:31:57.497+0000 7fd5bc14f200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T15:31:57.829 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:31:57.605+0000 7fd5bc14f200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T15:31:57.829 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:31:57.825+0000 7fd5bc14f200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-22T15:31:58.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:58 smithi145 ceph-mon[21899]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-04-22T15:31:58.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:58 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:58.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:58 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:58.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:58 smithi145 ceph-mon[26606]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-04-22T15:31:58.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:58 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:58.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:58 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:58.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:58 smithi172 ceph-mon[22747]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-04-22T15:31:58.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:58 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:58.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:58 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:58.718 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:31:58.382+0000 7fd5bc14f200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T15:31:58.719 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:31:58.481+0000 7fd5bc14f200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T15:31:58.996 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:31:58.749+0000 7fd5bc14f200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T15:31:58.996 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:31:58.891+0000 7fd5bc14f200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T15:31:58.996 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:31:58.992+0000 7fd5bc14f200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T15:31:59.308 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:59 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:31:59.116+0000 7fd5bc14f200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T15:31:59.605 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:59 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:31:59.605 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:59 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:59.605 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:31:59 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:59.605 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:31:59 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:31:59.450+0000 7fd5bc14f200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T15:31:59.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:59 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:31:59.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:59 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:59.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:31:59 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:59.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:59 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:31:59.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:59 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:31:59.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:31:59 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:00.359 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:32:00 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:32:00.080+0000 7fd5bc14f200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T15:32:00.359 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:32:00 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:32:00.312+0000 7fd5bc14f200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T15:32:00.359 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:32:00 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: [22/Apr/2024:15:32:00] ENGINE Bus STARTING 2024-04-22T15:32:00.628 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:00 smithi172 ceph-mon[22747]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:32:00.628 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:00 smithi172 ceph-mon[22747]: Standby manager daemon x restarted 2024-04-22T15:32:00.628 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:00 smithi172 ceph-mon[22747]: Standby manager daemon x started 2024-04-22T15:32:00.629 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:00 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.172:0/2371014163' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T15:32:00.629 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:00 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.172:0/2371014163' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:32:00.629 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:00 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.172:0/2371014163' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T15:32:00.629 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:00 smithi172 ceph-mon[22747]: from='mgr.? 172.21.15.172:0/2371014163' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:32:00.629 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:32:00 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: CherryPy Checker: 2024-04-22T15:32:00.629 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:32:00 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: The Application mounted at '' has an empty config. 2024-04-22T15:32:00.629 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:32:00 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:32:00.629 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:32:00 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: [22/Apr/2024:15:32:00] ENGINE Serving on http://:::9283 2024-04-22T15:32:00.629 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:32:00 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: [22/Apr/2024:15:32:00] ENGINE Bus STARTED 2024-04-22T15:32:00.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:00 smithi145 ceph-mon[21899]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:32:00.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:00 smithi145 ceph-mon[21899]: Standby manager daemon x restarted 2024-04-22T15:32:00.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:00 smithi145 ceph-mon[21899]: Standby manager daemon x started 2024-04-22T15:32:00.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:00 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.172:0/2371014163' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T15:32:00.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:00 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.172:0/2371014163' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:32:00.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:00 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.172:0/2371014163' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T15:32:00.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:00 smithi145 ceph-mon[21899]: from='mgr.? 172.21.15.172:0/2371014163' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:32:00.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:00 smithi145 ceph-mon[26606]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:32:00.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:00 smithi145 ceph-mon[26606]: Standby manager daemon x restarted 2024-04-22T15:32:00.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:00 smithi145 ceph-mon[26606]: Standby manager daemon x started 2024-04-22T15:32:00.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:00 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.172:0/2371014163' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T15:32:00.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:00 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.172:0/2371014163' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:32:00.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:00 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.172:0/2371014163' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T15:32:00.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:00 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.172:0/2371014163' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:32:02.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:02 smithi145 ceph-mon[26606]: mgrmap e43: y(active, since 27s), standbys: x 2024-04-22T15:32:02.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:02 smithi145 ceph-mon[26606]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:02.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:02 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:02.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:02 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:02.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:02 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:32:02.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:02 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:32:02.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:02 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:02.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:02 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:32:02.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:02 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:32:02.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:02 smithi145 ceph-mon[21899]: mgrmap e43: y(active, since 27s), standbys: x 2024-04-22T15:32:02.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:02 smithi145 ceph-mon[21899]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:02.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:02 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:02.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:02 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:02.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:02 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:32:02.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:02 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:32:02.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:02 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:02.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:02 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:32:02.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:02 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:32:02.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:02 smithi172 ceph-mon[22747]: mgrmap e43: y(active, since 27s), standbys: x 2024-04-22T15:32:02.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:02 smithi172 ceph-mon[22747]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:02.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:02 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:02.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:02 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:02.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:02 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:32:02.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:02 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:32:02.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:02 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:02.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:02 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:32:02.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:02 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:32:03.370 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-mon[21899]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 958 B/s rd, 0 op/s 2024-04-22T15:32:03.370 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-mon[21899]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:03.370 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-mon[21899]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:32:03.370 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-mon[21899]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:32:03.371 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-mon[26606]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 958 B/s rd, 0 op/s 2024-04-22T15:32:03.371 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-mon[26606]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:03.371 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-mon[26606]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:32:03.371 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-mon[26606]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:32:03.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:03 smithi172 ceph-mon[22747]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 958 B/s rd, 0 op/s 2024-04-22T15:32:03.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:03 smithi172 ceph-mon[22747]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:03.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:03 smithi172 ceph-mon[22747]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:32:03.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:03 smithi172 ceph-mon[22747]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:32:04.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:32:03.944+0000 7f42a94bd700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:32:04.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:32:04.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:32:04.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:32:04.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:32:04.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:04.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:04.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:32:04.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:32:04.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:32:04.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:32:04.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:32:04.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:32:04.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:32:04.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:32:04.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:32:04.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:32:04.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:32:04.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:32:04.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:04.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:04.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:32:04.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:32:04.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:32:04.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:32:04.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:32:04.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:32:04.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:32:04.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:32:04.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:32:04.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:32:04.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:32:04.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:32:04.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:32:04.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:32:04.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:32:04.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:32:04.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:32:04.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:32:04.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:32:04.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:32:04.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:32:04.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:32:04.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:04.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:04.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:32:04.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:32:04.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:32:04.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:32:04.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:32:04.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:32:04.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:32:04.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:32:04.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:32:04.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:32:04.128 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:32:04.128 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:32:04.128 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:04.128 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:04.612 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:32:04.612 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:04.612 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:04.612 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-22T15:32:04.612 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-22T15:32:04.612 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-22T15:32:04.612 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-22T15:32:04.612 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:04.612 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:04.612 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:04.613 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:04.613 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:04.613 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:04.613 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:04.613 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:04.613 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:04.613 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:04.613 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:04.613 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:04.613 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:04.613 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:04.613 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:32:04.614 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:04.614 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:04.614 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-22T15:32:04.614 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-22T15:32:04.614 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-22T15:32:04.614 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-22T15:32:04.614 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:04.614 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:04.614 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:04.614 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:04.614 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:04.614 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:04.614 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:04.614 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:04.614 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:04.615 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:04.615 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:04.615 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:04.615 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:04.615 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:04 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:04.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:04 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:32:04.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:04 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:04.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:04 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:04.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:04 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-22T15:32:04.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:04 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-22T15:32:04.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:04 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-22T15:32:04.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:04 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-22T15:32:04.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:04 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:04.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:04 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:04.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:04 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:04.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:04 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:04.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:04 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:04.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:04 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:04.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:04 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:04.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:04 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:04.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:04 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:04.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:04 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:04.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:04 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:04.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:04 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:04.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:04 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:04.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:04 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:05.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:05 smithi172 ceph-mon[22747]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:32:05.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:32:05.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:32:05.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:32:05.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:32:05.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:05.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:05.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:32:05.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:32:05.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:32:05.620 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:32:05.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:32:05.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:32:05.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:32:05.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:32:05.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:32:05.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:32:05.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:32:05.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:32:05.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:05.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:05.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:32:05.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:32:05.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:32:05.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:32:05.621 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:32:05.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:32:05.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:32:05.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:32:05.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:32:05.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:32:05.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:32:05.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:32:05.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:32:05.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:32:05.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:32:05.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:32:05.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:32:05.622 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:32:05.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:32:05.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:32:05.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:32:05.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:32:05.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:05.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:05.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:32:05.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:32:05.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:32:05.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:32:05.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:32:05.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:32:05.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:32:05.623 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:32:05.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:32:05.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:32:05.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:32:05.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:32:05.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:05.624 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:05.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:05 smithi172 ceph-mon[22747]: Upgrade: Setting container_image for all mgr 2024-04-22T15:32:05.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:05 smithi172 ceph-mon[22747]: Upgrade: Setting container_image for all crash 2024-04-22T15:32:05.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:05 smithi172 ceph-mon[22747]: Upgrade: Setting container_image for all mds 2024-04-22T15:32:05.624 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:05 smithi172 ceph-mon[22747]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T15:32:05.625 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:05 smithi172 ceph-mon[22747]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-22T15:32:05.625 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:05 smithi172 ceph-mon[22747]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T15:32:05.625 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:05 smithi172 ceph-mon[22747]: Upgrade: Setting container_image for all nfs 2024-04-22T15:32:05.625 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:05 smithi172 ceph-mon[22747]: Upgrade: Setting container_image for all nvmeof 2024-04-22T15:32:05.625 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:05 smithi172 ceph-mon[22747]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:05.625 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:05 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:32:05.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:05 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:32:05] "GET /metrics HTTP/1.1" 200 37996 "" "Prometheus/2.33.4" 2024-04-22T15:32:05.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:05 smithi145 ceph-mon[21899]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:32:05.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:32:05.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:32:05.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:32:05.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:32:05.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:05.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:05.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:32:05.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:32:05.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:32:05.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:32:05.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:32:05.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:32:05.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:32:05.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:32:05.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:32:05.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:32:05.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:32:05.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:32:05.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:05.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:05.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:32:05.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:32:05.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:32:05.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:32:05.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:32:05.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:32:05.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:32:05.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:32:05.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:32:05.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:32:05.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:32:05.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:32:05.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:32:05.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:32:05.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:32:05.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:32:05.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:32:05.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:32:05.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:32:05.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:32:05.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:32:05.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:32:05.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:05.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:05.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:32:05.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:32:05.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:32:05.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:32:05.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:32:05.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:32:05.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:32:05.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:32:05.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:32:05.631 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:05 smithi145 ceph-mon[26606]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:32:05.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:32:05.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:32:05.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:32:05.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:32:05.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:05.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:05.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:32:05.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:32:05.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:32:05.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:32:05.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:32:05.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:32:05.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:32:05.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:32:05.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:32:05.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:32:05.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:32:05.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:32:05.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:05.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:05.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:32:05.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:32:05.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:32:05.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:32:05.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:32:05.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:32:05.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:32:05.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:32:05.634 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:32:05.634 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:32:05.634 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:32:05.634 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:32:05.634 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:32:05.634 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:32:05.634 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:32:05.634 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:32:05.635 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:32:05.635 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:32:05.635 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:32:05.635 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:32:05.635 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:32:05.635 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:32:05.635 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:05.635 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:05.635 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:32:05.635 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:32:05.636 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:32:05.636 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:32:05.636 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:32:05.636 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:32:05.636 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:32:05.636 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:32:05.636 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:32:05.636 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:32:05.636 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:32:05.636 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:32:05.637 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:05.637 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:05.637 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:05 smithi145 ceph-mon[21899]: Upgrade: Setting container_image for all mgr 2024-04-22T15:32:05.637 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:05 smithi145 ceph-mon[21899]: Upgrade: Setting container_image for all crash 2024-04-22T15:32:05.637 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:05 smithi145 ceph-mon[21899]: Upgrade: Setting container_image for all mds 2024-04-22T15:32:05.637 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:05 smithi145 ceph-mon[21899]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T15:32:05.637 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:05 smithi145 ceph-mon[21899]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-22T15:32:05.637 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:05 smithi145 ceph-mon[21899]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T15:32:05.637 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:05 smithi145 ceph-mon[21899]: Upgrade: Setting container_image for all nfs 2024-04-22T15:32:05.637 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:05 smithi145 ceph-mon[21899]: Upgrade: Setting container_image for all nvmeof 2024-04-22T15:32:05.637 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:05 smithi145 ceph-mon[21899]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:05.638 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:05 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:32:05.638 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:32:05.638 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:32:05.638 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:32:05.638 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:05.638 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:05.638 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:05 smithi145 ceph-mon[26606]: Upgrade: Setting container_image for all mgr 2024-04-22T15:32:05.638 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:05 smithi145 ceph-mon[26606]: Upgrade: Setting container_image for all crash 2024-04-22T15:32:05.639 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:05 smithi145 ceph-mon[26606]: Upgrade: Setting container_image for all mds 2024-04-22T15:32:05.639 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:05 smithi145 ceph-mon[26606]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T15:32:05.639 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:05 smithi145 ceph-mon[26606]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-22T15:32:05.639 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:05 smithi145 ceph-mon[26606]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T15:32:05.639 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:05 smithi145 ceph-mon[26606]: Upgrade: Setting container_image for all nfs 2024-04-22T15:32:05.639 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:05 smithi145 ceph-mon[26606]: Upgrade: Setting container_image for all nvmeof 2024-04-22T15:32:05.639 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:05 smithi145 ceph-mon[26606]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:05.639 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:05 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:32:06.373 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:06 smithi145 systemd[1]: Stopping Ceph node-exporter.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:32:06.841 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:06 smithi145 podman[73508]: 2024-04-22 15:32:06.580963917 +0000 UTC m=+0.130381871 container died 7a7a9854f2474fb05f9a17fa7117a367f42f783bb1cb3b6d4ff4c62a8c8b679e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T15:32:07.122 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:06 smithi145 podman[73508]: 2024-04-22 15:32:06.936729163 +0000 UTC m=+0.486147116 container cleanup 7a7a9854f2474fb05f9a17fa7117a367f42f783bb1cb3b6d4ff4c62a8c8b679e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T15:32:07.123 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:06 smithi145 bash[73508]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a 2024-04-22T15:32:07.198 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:32:07.614 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:07 smithi145 ceph-mon[26606]: Upgrade: Updating node-exporter.a (1/2) 2024-04-22T15:32:07.614 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:07 smithi145 ceph-mon[26606]: Deploying daemon node-exporter.a on smithi145 2024-04-22T15:32:07.614 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:07 smithi145 ceph-mon[26606]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:07.614 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:07 smithi145 ceph-mon[21899]: Upgrade: Updating node-exporter.a (1/2) 2024-04-22T15:32:07.614 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:07 smithi145 ceph-mon[21899]: Deploying daemon node-exporter.a on smithi145 2024-04-22T15:32:07.614 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:07 smithi145 ceph-mon[21899]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:07.615 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:07 smithi145 podman[73522]: 2024-04-22 15:32:07.229915443 +0000 UTC m=+0.652813899 container remove 7a7a9854f2474fb05f9a17fa7117a367f42f783bb1cb3b6d4ff4c62a8c8b679e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T15:32:07.615 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:07 smithi145 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-22T15:32:07.615 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:07 smithi145 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-22T15:32:07.615 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:07 smithi145 systemd[1]: Stopped Ceph node-exporter.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:32:07.615 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:07 smithi145 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@node-exporter.a.service: Consumed 5.397s CPU time. 2024-04-22T15:32:07.615 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:07 smithi145 systemd[1]: Starting Ceph node-exporter.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:32:07.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:07 smithi172 ceph-mon[22747]: Upgrade: Updating node-exporter.a (1/2) 2024-04-22T15:32:07.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:07 smithi172 ceph-mon[22747]: Deploying daemon node-exporter.a on smithi145 2024-04-22T15:32:07.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:07 smithi172 ceph-mon[22747]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:07.872 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:07 smithi145 bash[73721]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-04-22T15:32:07.984 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:32:07.985 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (13m) 28s ago 21m 24.8M - 0.23.0 ba2b418f427c 2b762f668fa7 2024-04-22T15:32:07.985 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (13m) 10s ago 20m 54.0M - 8.3.5 dad864ee21e9 20a9f6cd1859 2024-04-22T15:32:07.985 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 error 28s ago 20m 59.4M - 3.5 e1d6a67b021e 9704631ffa0e 2024-04-22T15:32:07.985 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443,9283,8765 running (15s) 10s ago 25m 59.6M - 19.0.0-3223-g5883d494 3d983c26e278 e7500ef71604 2024-04-22T15:32:07.985 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (10m) 28s ago 27m 477M - 19.0.0-3223-g5883d494 3d983c26e278 b2de97566cf4 2024-04-22T15:32:07.985 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (27m) 28s ago 27m 77.9M 2048M 17.2.0 e1d6a67b021e 28106a7bf16e 2024-04-22T15:32:07.985 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (26m) 10s ago 26m 57.5M 2048M 17.2.0 e1d6a67b021e a2fc3297953a 2024-04-22T15:32:07.985 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (26m) 28s ago 26m 51.4M 2048M 17.2.0 e1d6a67b021e b71d9b9f8509 2024-04-22T15:32:07.985 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (13m) 28s ago 21m 16.2M - 1.3.1 1dbe0e931976 7a7a9854f247 2024-04-22T15:32:07.985 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (13m) 10s ago 21m 18.6M - 1.3.1 1dbe0e931976 e5b0a512c835 2024-04-22T15:32:07.985 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (25m) 28s ago 25m 55.8M 2992M 17.2.0 e1d6a67b021e ea9cdc639916 2024-04-22T15:32:07.986 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (25m) 28s ago 25m 53.9M 2992M 17.2.0 e1d6a67b021e 3b9b37a8781f 2024-04-22T15:32:07.986 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (24m) 28s ago 24m 50.7M 2992M 17.2.0 e1d6a67b021e d9502dded8f8 2024-04-22T15:32:07.986 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (24m) 28s ago 24m 51.8M 2992M 17.2.0 e1d6a67b021e 8c72f30e54fc 2024-04-22T15:32:07.986 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (23m) 10s ago 23m 57.6M 3248M 17.2.0 e1d6a67b021e 80001f97ed35 2024-04-22T15:32:07.986 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (23m) 10s ago 23m 54.2M 3248M 17.2.0 e1d6a67b021e ec3ab1335b5c 2024-04-22T15:32:07.986 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (23m) 10s ago 23m 54.6M 3248M 17.2.0 e1d6a67b021e 45d475edf1ca 2024-04-22T15:32:07.986 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (22m) 10s ago 22m 52.7M 3248M 17.2.0 e1d6a67b021e cc9e1d9e7916 2024-04-22T15:32:07.986 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (20s) 10s ago 22m 51.6M - 2.33.4 514e6a882f6e 83fc61bca238 2024-04-22T15:32:07.986 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (20m) 28s ago 20m 77.7M - 17.2.0 e1d6a67b021e c90a636c947e 2024-04-22T15:32:07.986 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (20m) 10s ago 20m 75.4M - 17.2.0 e1d6a67b021e 20ac46e9bfda 2024-04-22T15:32:08.384 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:32:08.384 INFO:teuthology.orchestra.run.smithi145.stdout: "mon": { 2024-04-22T15:32:08.384 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-22T15:32:08.385 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:32:08.385 INFO:teuthology.orchestra.run.smithi145.stdout: "mgr": { 2024-04-22T15:32:08.385 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:32:08.385 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:32:08.385 INFO:teuthology.orchestra.run.smithi145.stdout: "osd": { 2024-04-22T15:32:08.385 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T15:32:08.385 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:32:08.386 INFO:teuthology.orchestra.run.smithi145.stdout: "mds": {}, 2024-04-22T15:32:08.386 INFO:teuthology.orchestra.run.smithi145.stdout: "rgw": { 2024-04-22T15:32:08.386 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:32:08.386 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:32:08.386 INFO:teuthology.orchestra.run.smithi145.stdout: "overall": { 2024-04-22T15:32:08.386 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-22T15:32:08.387 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:32:08.387 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:32:08.387 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:32:08.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:08 smithi145 ceph-mon[26606]: from='client.25417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:32:08.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:08 smithi145 ceph-mon[21899]: from='client.25417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:32:08.626 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:08 smithi145 bash[73721]: Getting image source signatures 2024-04-22T15:32:08.626 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:08 smithi145 bash[73721]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-04-22T15:32:08.627 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:08 smithi145 bash[73721]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-04-22T15:32:08.627 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:08 smithi145 bash[73721]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-04-22T15:32:08.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:08 smithi172 ceph-mon[22747]: from='client.25417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:32:08.726 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:32:08.726 INFO:teuthology.orchestra.run.smithi145.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:32:08.726 INFO:teuthology.orchestra.run.smithi145.stdout: "in_progress": true, 2024-04-22T15:32:08.726 INFO:teuthology.orchestra.run.smithi145.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-04-22T15:32:08.726 INFO:teuthology.orchestra.run.smithi145.stdout: "services_complete": [ 2024-04-22T15:32:08.726 INFO:teuthology.orchestra.run.smithi145.stdout: "mgr" 2024-04-22T15:32:08.726 INFO:teuthology.orchestra.run.smithi145.stdout: ], 2024-04-22T15:32:08.726 INFO:teuthology.orchestra.run.smithi145.stdout: "progress": "2/2 daemons upgraded", 2024-04-22T15:32:08.727 INFO:teuthology.orchestra.run.smithi145.stdout: "message": "Currently upgrading node-exporter daemons", 2024-04-22T15:32:08.727 INFO:teuthology.orchestra.run.smithi145.stdout: "is_paused": false 2024-04-22T15:32:08.727 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:32:09.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:09 smithi145 ceph-mon[21899]: from='client.15501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:32:09.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:09 smithi145 ceph-mon[21899]: from='client.25429 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:32:09.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:09 smithi145 ceph-mon[21899]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:09.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:09 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:32:09.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:09 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3157278815' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:09.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:09 smithi145 ceph-mon[26606]: from='client.15501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:32:09.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:09 smithi145 ceph-mon[26606]: from='client.25429 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:32:09.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:09 smithi145 ceph-mon[26606]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:09.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:09 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:32:09.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:09 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3157278815' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:09.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:09 smithi172 ceph-mon[22747]: from='client.15501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:32:09.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:09 smithi172 ceph-mon[22747]: from='client.25429 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:32:09.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:09 smithi172 ceph-mon[22747]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:09.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:09 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:32:09.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:09 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/3157278815' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:10.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:10 smithi145 ceph-mon[26606]: from='client.25441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:32:10.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:10 smithi145 ceph-mon[21899]: from='client.25441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:32:10.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:10 smithi172 ceph-mon[22747]: from='client.25441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:32:11.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:11 smithi145 ceph-mon[21899]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 589 B/s rd, 0 op/s 2024-04-22T15:32:11.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:11 smithi145 ceph-mon[26606]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 589 B/s rd, 0 op/s 2024-04-22T15:32:11.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:11 smithi172 ceph-mon[22747]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 589 B/s rd, 0 op/s 2024-04-22T15:32:12.372 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:11 smithi145 bash[73721]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-04-22T15:32:12.373 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 bash[73721]: Writing manifest to image destination 2024-04-22T15:32:12.779 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 podman[73721]: 2024-04-22 15:32:12.393133038 +0000 UTC m=+4.679308865 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-22T15:32:12.779 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 podman[73721]: 2024-04-22 15:32:12.518748948 +0000 UTC m=+4.804924762 container create 8c75c26fec05f244883c540f215d8ad9667cf3608ba7133e8c33937e7a061839 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T15:32:12.780 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 podman[73721]: 2024-04-22 15:32:12.777167739 +0000 UTC m=+5.063343556 container init 8c75c26fec05f244883c540f215d8ad9667cf3608ba7133e8c33937e7a061839 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T15:32:13.123 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 podman[73721]: 2024-04-22 15:32:12.780637776 +0000 UTC m=+5.066813597 container start 8c75c26fec05f244883c540f215d8ad9667cf3608ba7133e8c33937e7a061839 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T15:32:13.123 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.784Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-22T15:32:13.123 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.784Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-22T15:32:13.123 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.785Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-22T15:32:13.123 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.785Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-22T15:32:13.123 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.785Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-22T15:32:13.124 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.785Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-22T15:32:13.124 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-22T15:32:13.124 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=arp 2024-04-22T15:32:13.124 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-22T15:32:13.124 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-22T15:32:13.124 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-22T15:32:13.124 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-22T15:32:13.124 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-22T15:32:13.124 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-22T15:32:13.124 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-22T15:32:13.124 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-22T15:32:13.125 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=edac 2024-04-22T15:32:13.125 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-22T15:32:13.125 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-22T15:32:13.125 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-22T15:32:13.125 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-22T15:32:13.125 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-22T15:32:13.125 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-22T15:32:13.125 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-22T15:32:13.125 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-22T15:32:13.125 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-22T15:32:13.126 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-22T15:32:13.126 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-22T15:32:13.126 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-22T15:32:13.126 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-22T15:32:13.126 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-22T15:32:13.129 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-22T15:32:13.129 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-22T15:32:13.130 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=os 2024-04-22T15:32:13.130 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-22T15:32:13.130 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-22T15:32:13.130 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-22T15:32:13.130 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-22T15:32:13.130 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-22T15:32:13.130 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-22T15:32:13.130 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-22T15:32:13.130 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=stat 2024-04-22T15:32:13.130 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-22T15:32:13.130 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-22T15:32:13.130 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-22T15:32:13.130 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=time 2024-04-22T15:32:13.130 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-22T15:32:13.131 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=uname 2024-04-22T15:32:13.131 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-22T15:32:13.131 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-22T15:32:13.131 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-22T15:32:13.131 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-22T15:32:13.131 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a[73820]: ts=2024-04-22T15:32:12.786Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-22T15:32:13.131 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 bash[73721]: 8c75c26fec05f244883c540f215d8ad9667cf3608ba7133e8c33937e7a061839 2024-04-22T15:32:13.131 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:32:12 smithi145 systemd[1]: Started Ceph node-exporter.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:32:13.610 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:13 smithi172 ceph-mon[22747]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:32:13.610 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:13 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:13.610 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:13 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:13.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:13 smithi145 ceph-mon[26606]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:32:13.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:13 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:13.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:13 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:13.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:13 smithi145 ceph-mon[21899]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:32:13.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:13 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:13.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:13 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:15.357 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:15 smithi172 systemd[1]: Stopping Ceph node-exporter.b for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:32:15.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:32:15] "GET /metrics HTTP/1.1" 200 37996 "" "Prometheus/2.33.4" 2024-04-22T15:32:15.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:15 smithi145 ceph-mon[21899]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:32:15.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:15 smithi145 ceph-mon[21899]: Upgrade: Updating node-exporter.b (2/2) 2024-04-22T15:32:15.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:15 smithi145 ceph-mon[21899]: Deploying daemon node-exporter.b on smithi172 2024-04-22T15:32:15.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:15 smithi145 ceph-mon[26606]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:32:15.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:15 smithi145 ceph-mon[26606]: Upgrade: Updating node-exporter.b (2/2) 2024-04-22T15:32:15.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:15 smithi145 ceph-mon[26606]: Deploying daemon node-exporter.b on smithi172 2024-04-22T15:32:15.718 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:15 smithi172 podman[59486]: 2024-04-22 15:32:15.460960426 +0000 UTC m=+0.105875337 container died e5b0a512c8352956d8f1bdfbb7df1bda1d18ba36d77f6db5a5898125246a62ed (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T15:32:15.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:15 smithi172 ceph-mon[22747]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:32:15.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:15 smithi172 ceph-mon[22747]: Upgrade: Updating node-exporter.b (2/2) 2024-04-22T15:32:15.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:15 smithi172 ceph-mon[22747]: Deploying daemon node-exporter.b on smithi172 2024-04-22T15:32:16.070 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:15 smithi172 podman[59486]: 2024-04-22 15:32:15.817084521 +0000 UTC m=+0.461999423 container cleanup e5b0a512c8352956d8f1bdfbb7df1bda1d18ba36d77f6db5a5898125246a62ed (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T15:32:16.070 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:15 smithi172 bash[59486]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b 2024-04-22T15:32:16.402 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:16 smithi172 podman[59500]: 2024-04-22 15:32:16.068091818 +0000 UTC m=+0.611168796 container remove e5b0a512c8352956d8f1bdfbb7df1bda1d18ba36d77f6db5a5898125246a62ed (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T15:32:16.402 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:16 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-22T15:32:16.402 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:16 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-22T15:32:16.403 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:16 smithi172 systemd[1]: Stopped Ceph node-exporter.b for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:32:16.403 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:16 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@node-exporter.b.service: Consumed 5.312s CPU time. 2024-04-22T15:32:16.718 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:16 smithi172 systemd[1]: Starting Ceph node-exporter.b for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:32:16.718 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:16 smithi172 bash[59633]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-04-22T15:32:17.219 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:16 smithi172 bash[59633]: Getting image source signatures 2024-04-22T15:32:17.219 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:16 smithi172 bash[59633]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-04-22T15:32:17.220 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:16 smithi172 bash[59633]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-04-22T15:32:17.220 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:16 smithi172 bash[59633]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-04-22T15:32:17.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:17 smithi145 ceph-mon[21899]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:17.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:17 smithi145 ceph-mon[26606]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:17.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:17 smithi172 ceph-mon[22747]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:18.968 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:18 smithi172 bash[59633]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-04-22T15:32:18.968 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:18 smithi172 bash[59633]: Writing manifest to image destination 2024-04-22T15:32:19.383 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-mon[22747]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:32:19.383 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:32:19.383 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 podman[59633]: 2024-04-22 15:32:19.055201006 +0000 UTC m=+2.559031536 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-22T15:32:19.383 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 podman[59633]: 2024-04-22 15:32:19.153373379 +0000 UTC m=+2.657203935 container create 8fd2949e2eaf0973ec1c44cca152ec6c96f8b6e9353482fab051598d8411a690 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T15:32:19.383 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 podman[59633]: 2024-04-22 15:32:19.380541184 +0000 UTC m=+2.884371699 container init 8fd2949e2eaf0973ec1c44cca152ec6c96f8b6e9353482fab051598d8411a690 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T15:32:19.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:19 smithi145 ceph-mon[21899]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:32:19.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:19 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:32:19.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:19 smithi145 ceph-mon[26606]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:32:19.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:19 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:32:19.719 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 podman[59633]: 2024-04-22 15:32:19.383444606 +0000 UTC m=+2.887275123 container start 8fd2949e2eaf0973ec1c44cca152ec6c96f8b6e9353482fab051598d8411a690 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T15:32:19.719 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.387Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-22T15:32:19.719 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.387Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-22T15:32:19.719 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.388Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-22T15:32:19.720 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.388Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-22T15:32:19.720 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-22T15:32:19.720 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-22T15:32:19.720 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-22T15:32:19.720 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=arp 2024-04-22T15:32:19.720 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-22T15:32:19.721 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-22T15:32:19.721 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-22T15:32:19.721 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-22T15:32:19.721 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-22T15:32:19.721 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-22T15:32:19.721 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-22T15:32:19.721 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-22T15:32:19.721 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=edac 2024-04-22T15:32:19.721 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-22T15:32:19.721 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-22T15:32:19.722 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-22T15:32:19.722 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-22T15:32:19.722 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-22T15:32:19.722 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-22T15:32:19.722 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-22T15:32:19.722 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-22T15:32:19.722 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-22T15:32:19.722 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-22T15:32:19.722 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-22T15:32:19.722 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-22T15:32:19.723 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-22T15:32:19.723 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-22T15:32:19.723 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-22T15:32:19.723 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-22T15:32:19.723 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=os 2024-04-22T15:32:19.723 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-22T15:32:19.723 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-22T15:32:19.723 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-22T15:32:19.724 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-22T15:32:19.724 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-22T15:32:19.724 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-22T15:32:19.724 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-22T15:32:19.724 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=stat 2024-04-22T15:32:19.724 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-22T15:32:19.724 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-22T15:32:19.724 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-22T15:32:19.725 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=time 2024-04-22T15:32:19.725 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-22T15:32:19.725 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=uname 2024-04-22T15:32:19.725 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-22T15:32:19.725 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-22T15:32:19.725 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.389Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-22T15:32:19.725 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.390Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-22T15:32:19.725 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b[59684]: ts=2024-04-22T15:32:19.390Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-22T15:32:19.726 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 bash[59633]: 8fd2949e2eaf0973ec1c44cca152ec6c96f8b6e9353482fab051598d8411a690 2024-04-22T15:32:19.726 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:32:19 smithi172 systemd[1]: Started Ceph node-exporter.b for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:32:20.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:20 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:32:20.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:20 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:20.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:20 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:20.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:20 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:32:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:20 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:32:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:20 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:20 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:20 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:32:20.676 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:20 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:32:20.677 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:20 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:20.677 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:20 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:20.677 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:20 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:32:21.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:21 smithi145 ceph-mon[21899]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:32:21.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:21 smithi145 ceph-mon[26606]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:32:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:21 smithi172 ceph-mon[22747]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:32:23.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:23 smithi145 ceph-mon[26606]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:23 smithi145 ceph-mon[21899]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:23.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:23 smithi172 ceph-mon[22747]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:25.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:25 smithi145 ceph-mon[21899]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:32:25.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:25 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:25.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:25 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:25.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:25 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:25.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:25 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:25.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:32:25] "GET /metrics HTTP/1.1" 200 37991 "" "Prometheus/2.33.4" 2024-04-22T15:32:25.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:25 smithi145 ceph-mon[26606]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:32:25.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:25 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:25.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:25 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:25.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:25 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:25.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:25 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:25.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:25 smithi172 ceph-mon[22747]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:32:25.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:25 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:25.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:25 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:25.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:25 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:25.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:25 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:26.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:26 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:26.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:26 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:26.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:26 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:26.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:26 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:26.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:26 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:26.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:26 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:26.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:26 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:26.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:26 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:26.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:26 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:26.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:26 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:26.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:26 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:26.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:26 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:27.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:27 smithi145 ceph-mon[21899]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:27.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:27 smithi145 ceph-mon[26606]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:27.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:27 smithi172 ceph-mon[22747]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:29.534 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:29 smithi145 ceph-mon[26606]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:32:29.534 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:29 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:32:29.534 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:29 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:29.534 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:29 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:29.534 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:29 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:29.534 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:29 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:29.534 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:29 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:32:29.534 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:29 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:32:29.534 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:29 smithi145 ceph-mon[26606]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 974 B/s rd, 0 op/s 2024-04-22T15:32:29.534 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:29 smithi145 ceph-mon[26606]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:29.535 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:29 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:29.535 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:29 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:32:29.535 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:29 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:32:29.535 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:29 smithi145 ceph-mon[21899]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:32:29.535 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:29 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:32:29.535 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:29 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:29.535 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:29 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:29.535 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:29 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:29.535 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:29 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:29.535 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:29 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:32:29.535 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:29 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:32:29.535 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:29 smithi145 ceph-mon[21899]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 974 B/s rd, 0 op/s 2024-04-22T15:32:29.536 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:29 smithi145 ceph-mon[21899]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:29.536 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:29 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:29.536 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:29 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:32:29.536 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:29 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:32:29.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:29 smithi172 ceph-mon[22747]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:32:29.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:29 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:32:29.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:29 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:29.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:29 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:29.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:29 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:29.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:29 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:29.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:29 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:32:29.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:29 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:32:29.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:29 smithi172 ceph-mon[22747]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 974 B/s rd, 0 op/s 2024-04-22T15:32:29.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:29 smithi172 ceph-mon[22747]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:29.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:29 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:29.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:29 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:32:29.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:29 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:32:30.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-mon[21899]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:32:30.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-mon[21899]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:32:30.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-mon[21899]: Health check update: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] (CEPHADM_FAILED_DAEMON) 2024-04-22T15:32:30.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:32:30.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:30.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:30.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:30.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:30.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:30.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:30.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:30.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:30.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:30.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-mon[26606]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:32:30.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-mon[26606]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:32:30.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-mon[26606]: Health check update: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] (CEPHADM_FAILED_DAEMON) 2024-04-22T15:32:30.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:32:30.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:30.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:30.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:30.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:30.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:30.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:30.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:30.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:30.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:30.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:32:30.402+0000 7f42a94bd700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:32:30.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:32:30.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:32:30.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:32:30.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:32:30.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:30.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:30.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:32:30.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:32:30.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:32:30.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:32:30.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:32:30.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:32:30.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:32:30.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:32:30.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:32:30.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:32:30.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:32:30.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:32:30.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:30.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:30.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:32:30.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:32:30.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:32:30.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:32:30.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:32:30.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:32:30.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:32:30.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:32:30.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:32:30.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:32:30.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:32:30.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:32:30.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:32:30.629 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:32:30.629 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:32:30.629 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:32:30.629 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:32:30.629 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:32:30.629 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:32:30.629 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:32:30.629 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:32:30.629 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:32:30.629 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:30.629 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:30.629 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:32:30.629 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:32:30.630 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:32:30.630 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:32:30.630 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:32:30.630 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:32:30.630 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:32:30.630 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:32:30.630 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:32:30.630 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:32:30.630 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:32:30.630 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:32:30.630 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:30.630 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:30.753 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:30 smithi172 ceph-mon[22747]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:32:30.753 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:30 smithi172 ceph-mon[22747]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:32:30.753 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:30 smithi172 ceph-mon[22747]: Health check update: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] (CEPHADM_FAILED_DAEMON) 2024-04-22T15:32:30.754 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:30 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:32:30.754 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:30 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:30.754 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:30 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:30.754 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:30 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:30.754 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:30 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:30.754 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:30 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:30.754 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:30 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:30.754 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:30 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:30.754 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:30 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:30.754 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:30 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:31 smithi172 ceph-mon[22747]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:32:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:32:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:32:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:32:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:32:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:32:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:32:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:32:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:32:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:32:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:32:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:32:31.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:32:31.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:32:31.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:32:31.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:32:31.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:32:31.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:31.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:31.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:32:31.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:32:31.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:32:31.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:32:31.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:32:31.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:32:31.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:32:31.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:32:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:32:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:32:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:32:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:32:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:32:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:32:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:32:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:32:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:32:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:32:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:32:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:32:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:32:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:32:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:31.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:32:31.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:32:31.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:32:31.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:32:31.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:32:31.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:32:31.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:32:31.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:32:31.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:32:31.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:32:31.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:32:31.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:32:31.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:31.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:31.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:31 smithi172 ceph-mon[22747]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:31.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:31 smithi145 ceph-mon[21899]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:32:31.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:32:31.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:32:31.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:32:31.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:32:31.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:31.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:31.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:32:31.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:32:31.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:32:31.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:32:31.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:32:31.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:32:31.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:32:31.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:32:31.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:32:31.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:32:31.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:32:31.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:32:31.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:31.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:31.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:32:31.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:32:31.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:32:31.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:32:31.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:32:31.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:32:31.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:32:31.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:32:31.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:32:31.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:32:31.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:32:31.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:32:31.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:32:31.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:32:31.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:32:31.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:32:31.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:32:31.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:32:31.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:32:31.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:32:31.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:32:31.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:32:31.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:31.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:31.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:32:31.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:32:31.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:32:31.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:32:31.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:32:31.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:32:31.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:32:31.877 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:32:31.877 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:32:31.877 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:31 smithi145 ceph-mon[26606]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:32:31.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:32:31.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:32:31.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:32:31.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:32:31.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:31.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:31.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:32:31.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:32:31.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:32:31.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:32:31.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:32:31.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:32:31.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:32:31.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:32:31.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:32:31.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:32:31.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:32:31.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:32:31.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:31.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:31.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:32:31.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:32:31.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:32:31.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:32:31.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:32:31.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:32:31.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:32:31.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:32:31.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:32:31.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:32:31.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:32:31.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:32:31.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:32:31.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:32:31.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:32:31.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:32:31.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:32:31.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:32:31.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:32:31.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:32:31.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:32:31.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:32:31.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:31.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:31.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:32:31.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:32:31.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:32:31.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:32:31.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:32:31.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:32:31.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:32:31.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:32:31.881 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:32:31.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:32:31.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:32:31.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:32:31.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:31.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:31.881 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:31 smithi145 ceph-mon[21899]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:31.881 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:32:31.881 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:32:31.882 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:32:31.882 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:31.882 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:31.882 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:31 smithi145 ceph-mon[26606]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:32.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:32 smithi145 ceph-mon[21899]: Upgrade: Updating prometheus.a 2024-04-22T15:32:32.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:32 smithi145 ceph-mon[21899]: Deploying daemon prometheus.a on smithi172 2024-04-22T15:32:32.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:32 smithi145 ceph-mon[21899]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:32.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:32 smithi145 ceph-mon[26606]: Upgrade: Updating prometheus.a 2024-04-22T15:32:32.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:32 smithi145 ceph-mon[26606]: Deploying daemon prometheus.a on smithi172 2024-04-22T15:32:32.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:32 smithi145 ceph-mon[26606]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:32.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:32 smithi172 ceph-mon[22747]: Upgrade: Updating prometheus.a 2024-04-22T15:32:32.980 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:32 smithi172 ceph-mon[22747]: Deploying daemon prometheus.a on smithi172 2024-04-22T15:32:32.980 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:32 smithi172 ceph-mon[22747]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:35.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:35 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:32:35.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:35 smithi145 ceph-mon[21899]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 601 B/s rd, 0 op/s 2024-04-22T15:32:35.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:35 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:32:35.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:35 smithi145 ceph-mon[26606]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 601 B/s rd, 0 op/s 2024-04-22T15:32:35.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:32:35] "GET /metrics HTTP/1.1" 200 37996 "" "Prometheus/2.33.4" 2024-04-22T15:32:35.670 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:35 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:32:35.670 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:35 smithi172 ceph-mon[22747]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 601 B/s rd, 0 op/s 2024-04-22T15:32:36.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:36 smithi145 ceph-mon[21899]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:36.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:36 smithi145 ceph-mon[26606]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:36.890 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:36 smithi172 ceph-mon[22747]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:38.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:38 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:32:38.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:38 smithi145 ceph-mon[26606]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:32:38.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:38 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:32:38.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:38 smithi145 ceph-mon[21899]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:32:38.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:38 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:32:38.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:38 smithi172 ceph-mon[22747]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:32:39.087 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:32:39.742 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:32:39.743 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (13m) 15s ago 21m 24.8M - 0.23.0 ba2b418f427c 2b762f668fa7 2024-04-22T15:32:39.743 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (13m) 15s ago 21m 53.5M - 8.3.5 dad864ee21e9 20a9f6cd1859 2024-04-22T15:32:39.743 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 unknown 15s ago 20m 59.6M - 3.5 e1d6a67b021e 9704631ffa0e 2024-04-22T15:32:39.743 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443,9283,8765 running (47s) 15s ago 26m 451M - 19.0.0-3223-g5883d494 3d983c26e278 e7500ef71604 2024-04-22T15:32:39.743 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (10m) 15s ago 27m 501M - 19.0.0-3223-g5883d494 3d983c26e278 b2de97566cf4 2024-04-22T15:32:39.743 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (28m) 15s ago 28m 78.3M 2048M 17.2.0 e1d6a67b021e 28106a7bf16e 2024-04-22T15:32:39.743 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (26m) 15s ago 26m 58.5M 2048M 17.2.0 e1d6a67b021e a2fc3297953a 2024-04-22T15:32:39.743 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (26m) 15s ago 26m 56.5M 2048M 17.2.0 e1d6a67b021e b71d9b9f8509 2024-04-22T15:32:39.743 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (27s) 15s ago 22m 8028k - 1.5.0 0da6a335fe13 8c75c26fec05 2024-04-22T15:32:39.743 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (20s) 15s ago 22m 3330k - 1.5.0 0da6a335fe13 8fd2949e2eaf 2024-04-22T15:32:39.743 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (26m) 15s ago 26m 56.7M 2992M 17.2.0 e1d6a67b021e ea9cdc639916 2024-04-22T15:32:39.743 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (25m) 15s ago 25m 55.2M 2992M 17.2.0 e1d6a67b021e 3b9b37a8781f 2024-04-22T15:32:39.744 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (25m) 15s ago 25m 50.8M 2992M 17.2.0 e1d6a67b021e d9502dded8f8 2024-04-22T15:32:39.744 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (24m) 15s ago 24m 53.0M 2992M 17.2.0 e1d6a67b021e 8c72f30e54fc 2024-04-22T15:32:39.744 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (24m) 15s ago 24m 57.7M 3248M 17.2.0 e1d6a67b021e 80001f97ed35 2024-04-22T15:32:39.744 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (24m) 15s ago 24m 54.2M 3248M 17.2.0 e1d6a67b021e ec3ab1335b5c 2024-04-22T15:32:39.744 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (23m) 15s ago 23m 54.6M 3248M 17.2.0 e1d6a67b021e 45d475edf1ca 2024-04-22T15:32:39.744 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (23m) 15s ago 23m 52.8M 3248M 17.2.0 e1d6a67b021e cc9e1d9e7916 2024-04-22T15:32:39.744 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (52s) 15s ago 22m 64.1M - 2.33.4 514e6a882f6e 83fc61bca238 2024-04-22T15:32:39.744 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (21m) 15s ago 21m 77.9M - 17.2.0 e1d6a67b021e c90a636c947e 2024-04-22T15:32:39.744 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (20m) 15s ago 20m 75.5M - 17.2.0 e1d6a67b021e 20ac46e9bfda 2024-04-22T15:32:40.144 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:32:40.144 INFO:teuthology.orchestra.run.smithi145.stdout: "mon": { 2024-04-22T15:32:40.144 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-22T15:32:40.144 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:32:40.144 INFO:teuthology.orchestra.run.smithi145.stdout: "mgr": { 2024-04-22T15:32:40.144 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:32:40.145 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:32:40.145 INFO:teuthology.orchestra.run.smithi145.stdout: "osd": { 2024-04-22T15:32:40.145 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T15:32:40.145 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:32:40.145 INFO:teuthology.orchestra.run.smithi145.stdout: "mds": {}, 2024-04-22T15:32:40.145 INFO:teuthology.orchestra.run.smithi145.stdout: "rgw": { 2024-04-22T15:32:40.145 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:32:40.145 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:32:40.145 INFO:teuthology.orchestra.run.smithi145.stdout: "overall": { 2024-04-22T15:32:40.145 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-22T15:32:40.145 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:32:40.145 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:32:40.145 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:32:40.409 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:40 smithi145 ceph-mon[21899]: from='client.15525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:32:40.409 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:40 smithi145 ceph-mon[21899]: from='client.15531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:32:40.410 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:40 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2043684598' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:40.410 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:40 smithi145 ceph-mon[26606]: from='client.15525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:32:40.410 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:40 smithi145 ceph-mon[26606]: from='client.15531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:32:40.410 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:40 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2043684598' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:40.488 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:32:40.488 INFO:teuthology.orchestra.run.smithi145.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:32:40.488 INFO:teuthology.orchestra.run.smithi145.stdout: "in_progress": true, 2024-04-22T15:32:40.488 INFO:teuthology.orchestra.run.smithi145.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-04-22T15:32:40.489 INFO:teuthology.orchestra.run.smithi145.stdout: "services_complete": [ 2024-04-22T15:32:40.489 INFO:teuthology.orchestra.run.smithi145.stdout: "mgr" 2024-04-22T15:32:40.489 INFO:teuthology.orchestra.run.smithi145.stdout: ], 2024-04-22T15:32:40.489 INFO:teuthology.orchestra.run.smithi145.stdout: "progress": "2/2 daemons upgraded", 2024-04-22T15:32:40.489 INFO:teuthology.orchestra.run.smithi145.stdout: "message": "Currently upgrading prometheus daemons", 2024-04-22T15:32:40.489 INFO:teuthology.orchestra.run.smithi145.stdout: "is_paused": false 2024-04-22T15:32:40.489 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:32:40.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:40 smithi172 ceph-mon[22747]: from='client.15525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:32:40.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:40 smithi172 ceph-mon[22747]: from='client.15531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:32:40.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:40 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2043684598' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:41.611 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:41 smithi172 ceph-mon[22747]: from='client.15537 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:32:41.611 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:41 smithi172 ceph-mon[22747]: from='client.15549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:32:41.611 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:41 smithi172 ceph-mon[22747]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:41.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:41 smithi145 ceph-mon[21899]: from='client.15537 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:32:41.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:41 smithi145 ceph-mon[21899]: from='client.15549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:32:41.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:41 smithi145 ceph-mon[21899]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:41.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:41 smithi145 ceph-mon[26606]: from='client.15537 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:32:41.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:41 smithi145 ceph-mon[26606]: from='client.15549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:32:41.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:41 smithi145 ceph-mon[26606]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:42.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:42 smithi145 ceph-mon[21899]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:32:42.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:42 smithi145 ceph-mon[26606]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:32:42.877 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:42 smithi172 ceph-mon[22747]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:32:43.541 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:43 smithi172 systemd[1]: Stopping Ceph prometheus.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:32:43.873 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:43 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:32:43.538Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-22T15:32:43.874 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:43 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:32:43.538Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-22T15:32:43.874 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:43 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:32:43.538Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-22T15:32:43.874 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:43 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:32:43.538Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-22T15:32:43.874 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:43 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:32:43.538Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-22T15:32:43.874 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:43 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:32:43.538Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-22T15:32:43.874 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:43 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:32:43.538Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-22T15:32:43.874 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:43 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:32:43.538Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-22T15:32:43.874 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:43 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:32:43.539Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-22T15:32:43.874 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:43 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:32:43.539Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-22T15:32:43.874 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:43 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:32:43.539Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-22T15:32:43.874 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:43 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[57590]: ts=2024-04-22T15:32:43.539Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-22T15:32:43.874 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:43 smithi172 podman[61184]: 2024-04-22 15:32:43.54870424 +0000 UTC m=+0.142604064 container died 83fc61bca2388aead7c3af6214ec4cef1bf55b417be7ddeec9a5fe3a3f6f919f (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:32:43.874 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:43 smithi172 podman[61184]: 2024-04-22 15:32:43.871163215 +0000 UTC m=+0.465063041 container cleanup 83fc61bca2388aead7c3af6214ec4cef1bf55b417be7ddeec9a5fe3a3f6f919f (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:32:43.875 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:43 smithi172 bash[61184]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a 2024-04-22T15:32:44.217 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:44 smithi172 podman[61198]: 2024-04-22 15:32:44.055594827 +0000 UTC m=+0.507494451 container remove 83fc61bca2388aead7c3af6214ec4cef1bf55b417be7ddeec9a5fe3a3f6f919f (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:32:44.717 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:44 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-22T15:32:44.717 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:44 smithi172 systemd[1]: Stopped Ceph prometheus.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:32:44.717 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:44 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.300s CPU time. 2024-04-22T15:32:44.718 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:44 smithi172 systemd[1]: Starting Ceph prometheus.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:32:44.968 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:44 smithi172 podman[61332]: 2024-04-22 15:32:44.715207593 +0000 UTC m=+0.114088068 container create 51cf4df6e8d45c7decc16f595f43cf242f20c56a1634d623e04631b85216f6e8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:32:44.969 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:44 smithi172 podman[61332]: 2024-04-22 15:32:44.620683608 +0000 UTC m=+0.019564041 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-22T15:32:44.969 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:44 smithi172 podman[61332]: 2024-04-22 15:32:44.917388294 +0000 UTC m=+0.316268727 container init 51cf4df6e8d45c7decc16f595f43cf242f20c56a1634d623e04631b85216f6e8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:32:44.969 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:44 smithi172 podman[61332]: 2024-04-22 15:32:44.91982167 +0000 UTC m=+0.318702106 container start 51cf4df6e8d45c7decc16f595f43cf242f20c56a1634d623e04631b85216f6e8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:32:44.969 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:32:44.943Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-22T15:32:44.969 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:32:44.943Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-22T15:32:44.969 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:32:44.943Z caller=main.go:570 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi172 (none))" 2024-04-22T15:32:44.969 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:32:44.943Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-22T15:32:44.969 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:32:44.943Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-22T15:32:44.969 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:32:44.945Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-22T15:32:44.969 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:32:44.945Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-22T15:32:44.969 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:32:44.946Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-22T15:32:44.969 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:32:44.946Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-22T15:32:44.969 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:32:44.948Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-22T15:32:44.970 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:32:44.948Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.668µs 2024-04-22T15:32:44.970 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:32:44.948Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-22T15:32:44.970 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:32:44.955Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=6 2024-04-22T15:32:45.333 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:32:44.967Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=6 2024-04-22T15:32:45.333 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:32:44.978Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=6 2024-04-22T15:32:45.333 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:32:44.980Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=6 2024-04-22T15:32:45.333 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:32:44.982Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=6 2024-04-22T15:32:45.333 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:32:44.983Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=6 2024-04-22T15:32:45.333 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:32:44.983Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=6 maxSegment=6 2024-04-22T15:32:45.333 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:32:44.983Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=28.975µs wal_replay_duration=34.942484ms wbl_replay_duration=103ns total_replay_duration=34.989368ms 2024-04-22T15:32:45.333 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:32:44.987Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-22T15:32:45.334 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:32:44.987Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-22T15:32:45.334 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:32:44.987Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-22T15:32:45.334 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:45 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:32:45.008Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=21.019736ms db_storage=831ns remote_storage=1.144µs web_handler=424ns query_engine=965ns scrape=261.671µs scrape_sd=89.308µs notify=18.299µs notify_sd=15.991µs rules=20.323199ms tracing=6.395µs 2024-04-22T15:32:45.334 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:45 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:32:45.008Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-22T15:32:45.334 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:45 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:32:45.008Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-22T15:32:45.334 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:45 smithi172 bash[61332]: 51cf4df6e8d45c7decc16f595f43cf242f20c56a1634d623e04631b85216f6e8 2024-04-22T15:32:45.334 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:32:45 smithi172 systemd[1]: Started Ceph prometheus.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:32:45.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:45 smithi145 ceph-mon[26606]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:32:45.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:45 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:45.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:45 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:45.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:45 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:32:45.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:45 smithi145 ceph-mon[21899]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:32:45.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:45 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:45.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:45 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:45.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:45 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:32:45.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:45 smithi172 ceph-mon[22747]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:32:45.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:45 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:45.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:45 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:45.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:45 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:32:46.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:46 smithi145 ceph-mon[21899]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:46.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:46 smithi145 ceph-mon[26606]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:46.951 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:46 smithi172 ceph-mon[22747]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:48.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:48 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:32:48.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:48 smithi145 ceph-mon[21899]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:32:48.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:48 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:32:48.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:48 smithi145 ceph-mon[26606]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:32:48.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:48 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:32:48.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:48 smithi172 ceph-mon[22747]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:32:50.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:50 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:50.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:50 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:50.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:50 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:32:50.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:50 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:50.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:50 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:50.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:50 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:32:50.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:50 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:50.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:50 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:50.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:50 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:32:51.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:51 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:51.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:51 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:51.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:51 smithi145 ceph-mon[21899]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:51.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:51 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:51.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:51 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:51.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:51 smithi145 ceph-mon[26606]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:51.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:51 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:51.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:51 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:51.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:51 smithi172 ceph-mon[22747]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:32:52.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:52 smithi172 ceph-mon[22747]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:32:53.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:52 smithi145 ceph-mon[21899]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:32:53.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:52 smithi145 ceph-mon[26606]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:32:54.669 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:54 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:54.669 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:54 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:54.669 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:54 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:32:54.669 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:54 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:32:54.669 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:54 smithi145 ceph-mon[26606]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 920 B/s rd, 0 op/s 2024-04-22T15:32:54.669 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:54 smithi145 ceph-mon[26606]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:32:54.669 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:54 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:54.669 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:54 smithi145 ceph-mon[26606]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:32:54.669 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:54 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:32:54.669 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:54 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:32:54.670 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:54 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:54.670 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:54 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:54.670 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:54 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:32:54.670 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:54 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:32:54.670 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:54 smithi145 ceph-mon[21899]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 920 B/s rd, 0 op/s 2024-04-22T15:32:54.670 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:54 smithi145 ceph-mon[21899]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:32:54.670 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:54 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:54.670 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:54 smithi145 ceph-mon[21899]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:32:54.670 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:54 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:32:54.670 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:54 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:32:54.670 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:54 smithi145 ceph-mon[21899]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:32:54.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:54 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:54.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:54 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:54.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:54 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:32:54.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:54 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:32:54.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:54 smithi172 ceph-mon[22747]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 920 B/s rd, 0 op/s 2024-04-22T15:32:54.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:54 smithi172 ceph-mon[22747]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:32:54.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:54 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:32:54.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:54 smithi172 ceph-mon[22747]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:32:54.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:54 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:32:54.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:54 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:32:54.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:54 smithi172 ceph-mon[22747]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:32:55.122 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:54 smithi145 ceph-mon[26606]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:32:55.550 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:32:55] "GET /metrics HTTP/1.1" 200 37991 "" "Prometheus/2.43.0" 2024-04-22T15:32:55.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:32:55.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:32:55.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:55.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:55.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:55.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:55.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:55.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:55.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:55.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:55.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:55.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:55.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:32:55.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:32:55.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:55.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:55.875 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:55.875 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:55.875 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:55.875 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:55.875 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:55.875 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:55.875 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:55.875 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:55.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:32:55.630+0000 7f42a94bd700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:32:55.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:32:55.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:32:55.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:32:55.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:32:55.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:55.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:55.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:32:55.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:32:55.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:32:55.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:32:55.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:32:55.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:32:55.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:32:55.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:32:55.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:32:55.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:32:55.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:32:55.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:32:55.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:55.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:55.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:32:55.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:32:55.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:32:55.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:32:55.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:32:55.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:32:55.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:32:55.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:32:55.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:32:55.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:32:55.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:32:55.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:32:55.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:32:55.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:32:55.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:32:55.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:32:55.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:32:55.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:32:55.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:32:55.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:32:55.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:32:55.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:32:55.879 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:55.879 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:55.879 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:32:55.879 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:32:55.879 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:32:55.879 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:32:55.879 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:32:55.879 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:32:55.879 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:32:55.879 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:32:55.879 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:32:55.879 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:32:55.879 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:32:55.879 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:32:55.879 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:55.880 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:32:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:55.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:55 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:32:55.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:55 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:32:55.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:55 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:55.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:55 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:55.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:55 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:55.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:55 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:55.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:55 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:55.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:55 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:55.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:55 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:55.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:55 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:55.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:55 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:55.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:55 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:32:56.840 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:56 smithi145 ceph-mon[26606]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:32:56.840 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:32:56.841 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:32:56.841 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:32:56.841 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:32:56.841 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:56.841 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:56.841 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:32:56.841 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:32:56.841 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:32:56.841 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:32:56.841 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:32:56.841 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:32:56.841 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:32:56.841 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:32:56.841 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:32:56.842 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:32:56.842 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:32:56.842 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:32:56.842 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:56.842 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:56.842 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:32:56.842 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:32:56.842 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:32:56.842 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:32:56.842 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:32:56.842 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:32:56.842 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:32:56.842 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:32:56.842 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:32:56.842 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:32:56.843 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:32:56.843 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:32:56.843 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:32:56.843 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:32:56.843 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:32:56.843 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:32:56.843 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:32:56.843 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:32:56.843 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:32:56.843 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:32:56.843 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:32:56.843 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:32:56.843 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:56.843 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:56.844 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:32:56.844 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:32:56.844 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:32:56.844 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:32:56.844 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:32:56.844 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:32:56.844 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:32:56.844 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:32:56.844 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:32:56.845 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:56 smithi145 ceph-mon[21899]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:32:56.845 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:32:56.845 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:32:56.845 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:32:56.845 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:32:56.845 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:56.845 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:56.845 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:32:56.845 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:32:56.845 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:32:56.846 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:32:56.846 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:32:56.846 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:32:56.846 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:32:56.846 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:32:56.846 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:32:56.846 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:32:56.846 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:32:56.846 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:32:56.846 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:56.846 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:56.846 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:32:56.846 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:32:56.847 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:32:56.847 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:32:56.847 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:32:56.847 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:32:56.847 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:32:56.847 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:32:56.847 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:32:56.847 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:32:56.847 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:32:56.847 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:32:56.848 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:32:56.848 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:32:56.848 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:32:56.848 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:32:56.848 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:32:56.848 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:32:56.848 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:32:56.848 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:32:56.848 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:32:56.848 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:32:56.848 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:56.848 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:56.848 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:32:56.848 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:32:56.849 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:32:56.849 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:32:56.849 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:32:56.849 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:32:56.849 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:32:56.849 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:32:56.849 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:32:56.849 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:32:56.849 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:32:56.849 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:32:56.849 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:56.849 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:56.850 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:56 smithi145 ceph-mon[26606]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:32:56.850 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:56 smithi145 ceph-mon[26606]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:32:56.850 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:32:56.850 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:32:56.850 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:32:56.850 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:56.850 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:56.850 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:56 smithi145 ceph-mon[21899]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:32:56.850 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:56 smithi145 ceph-mon[21899]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:32:56.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:56 smithi172 ceph-mon[22747]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:32:56.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:32:56.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:32:56.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:32:56.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:32:56.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:56.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:56.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:32:56.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:32:56.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:32:56.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:32:56.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:32:56.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:32:56.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:32:56.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:32:56.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:32:56.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:32:56.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:32:56.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:32:56.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:56.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:56.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:32:56.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:32:56.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:32:56.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:32:56.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:32:56.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:32:56.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:32:56.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:32:56.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:32:56.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:32:56.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:32:56.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:32:56.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:32:56.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:32:56.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:32:56.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:32:56.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:32:56.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:32:56.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:32:56.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:32:56.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:32:56.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:32:56.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:56.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:56.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:32:56.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:32:56.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:32:56.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:32:56.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:32:56.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:32:56.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:32:56.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:32:56.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:32:56.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:32:56.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:32:56.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:32:56.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:32:56.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:32:56.972 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:56 smithi172 ceph-mon[22747]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T15:32:56.972 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:56 smithi172 ceph-mon[22747]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:32:57.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:57 smithi172 ceph-mon[22747]: Upgrade: Updating alertmanager.a 2024-04-22T15:32:57.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:57 smithi172 ceph-mon[22747]: Deploying daemon alertmanager.a on smithi145 2024-04-22T15:32:58.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:57 smithi145 ceph-mon[21899]: Upgrade: Updating alertmanager.a 2024-04-22T15:32:58.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:57 smithi145 ceph-mon[21899]: Deploying daemon alertmanager.a on smithi145 2024-04-22T15:32:58.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:57 smithi145 ceph-mon[26606]: Upgrade: Updating alertmanager.a 2024-04-22T15:32:58.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:57 smithi145 ceph-mon[26606]: Deploying daemon alertmanager.a on smithi145 2024-04-22T15:32:59.122 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:58 smithi145 ceph-mon[26606]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:32:59.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:32:58 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:32:59.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:58 smithi145 ceph-mon[21899]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:32:59.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:32:58 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:32:59.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:58 smithi172 ceph-mon[22747]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:32:59.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:32:58 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:33:01.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:01 smithi145 ceph-mon[21899]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 561 B/s rd, 0 op/s 2024-04-22T15:33:01.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:01 smithi145 ceph-mon[26606]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 561 B/s rd, 0 op/s 2024-04-22T15:33:01.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:01 smithi172 ceph-mon[22747]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 561 B/s rd, 0 op/s 2024-04-22T15:33:03.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:03 smithi145 ceph-mon[21899]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:33:03.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:03 smithi145 ceph-mon[26606]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:33:03.655 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:03 smithi172 ceph-mon[22747]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:33:05.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:05 smithi172 ceph-mon[22747]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:33:05.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:05 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:33:05.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:05 smithi145 ceph-mon[26606]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:33:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:05 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:33:05.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:05 smithi145 ceph-mon[21899]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:33:05.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:05 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:33:05.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:05 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:33:05] "GET /metrics HTTP/1.1" 200 37994 "" "Prometheus/2.43.0" 2024-04-22T15:33:06.480 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:33:06 smithi145 systemd[1]: Stopping Ceph alertmanager.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:33:06.839 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:33:06 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[56207]: level=info ts=2024-04-22T15:33:06.477Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-04-22T15:33:06.839 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:33:06 smithi145 podman[75809]: 2024-04-22 15:33:06.558846617 +0000 UTC m=+0.232580893 container died 2b762f668fa703842a8b08ef93b0a393886d5d72787fe3259e3b56c26eca2399 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T15:33:07.122 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:33:06 smithi145 podman[75809]: 2024-04-22 15:33:06.994296422 +0000 UTC m=+0.668030698 container cleanup 2b762f668fa703842a8b08ef93b0a393886d5d72787fe3259e3b56c26eca2399 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T15:33:07.122 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:33:06 smithi145 bash[75809]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a 2024-04-22T15:33:07.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:07 smithi145 ceph-mon[21899]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:33:07.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:07 smithi145 ceph-mon[26606]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:33:07.623 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:33:07 smithi145 podman[75823]: 2024-04-22 15:33:07.262870423 +0000 UTC m=+0.703419030 container remove 2b762f668fa703842a8b08ef93b0a393886d5d72787fe3259e3b56c26eca2399 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T15:33:07.623 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:33:07 smithi145 podman[75823]: 2024-04-22 15:33:07.372106527 +0000 UTC m=+0.812655136 volume remove b7895904af97554f62e01b94b5990cdffae4f0ef93cb98e2fed4b6aa4d96e495 2024-04-22T15:33:07.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:07 smithi172 ceph-mon[22747]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:33:08.084 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:33:07 smithi145 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-22T15:33:08.084 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:33:07 smithi145 systemd[1]: Stopped Ceph alertmanager.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:33:08.084 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:33:07 smithi145 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@alertmanager.a.service: Consumed 1.492s CPU time. 2024-04-22T15:33:08.084 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:33:07 smithi145 systemd[1]: Starting Ceph alertmanager.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:33:08.372 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:33:08 smithi145 podman[75951]: 2024-04-22 15:33:07.984684464 +0000 UTC m=+0.018094095 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-04-22T15:33:08.373 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:33:08 smithi145 podman[75951]: 2024-04-22 15:33:08.082101358 +0000 UTC m=+0.115510983 volume create 97b5e88a6b2a7d0eb86406430ccb6b8aeb758f9b485daa2efc8978c209097f67 2024-04-22T15:33:08.373 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:33:08 smithi145 podman[75951]: 2024-04-22 15:33:08.191067044 +0000 UTC m=+0.224476669 container create 1c58ccf37ffd79a19f1e1f23b6570349cfd57ec0e4f0e289d8a172d47e89c0b1 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T15:33:08.873 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:33:08 smithi145 podman[75951]: 2024-04-22 15:33:08.493820506 +0000 UTC m=+0.527230133 container init 1c58ccf37ffd79a19f1e1f23b6570349cfd57ec0e4f0e289d8a172d47e89c0b1 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T15:33:08.873 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:33:08 smithi145 podman[75951]: 2024-04-22 15:33:08.496429976 +0000 UTC m=+0.529839608 container start 1c58ccf37ffd79a19f1e1f23b6570349cfd57ec0e4f0e289d8a172d47e89c0b1 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T15:33:08.873 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:33:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[75967]: ts=2024-04-22T15:33:08.508Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-22T15:33:08.873 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:33:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[75967]: ts=2024-04-22T15:33:08.508Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-22T15:33:08.873 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:33:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[75967]: ts=2024-04-22T15:33:08.509Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.145 port=9094 2024-04-22T15:33:08.873 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:33:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[75967]: ts=2024-04-22T15:33:08.511Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-22T15:33:08.873 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:33:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[75967]: ts=2024-04-22T15:33:08.533Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-22T15:33:08.873 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:33:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[75967]: ts=2024-04-22T15:33:08.534Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-22T15:33:08.873 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:33:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[75967]: ts=2024-04-22T15:33:08.537Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-22T15:33:08.873 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:33:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[75967]: ts=2024-04-22T15:33:08.537Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-22T15:33:08.874 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:33:08 smithi145 bash[75951]: 1c58ccf37ffd79a19f1e1f23b6570349cfd57ec0e4f0e289d8a172d47e89c0b1 2024-04-22T15:33:08.874 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:33:08 smithi145 systemd[1]: Started Ceph alertmanager.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:33:09.596 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:09 smithi145 ceph-mon[26606]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:33:09.596 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:09 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:33:09.596 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:09 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:09.596 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:09 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:09.596 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:09 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:33:09.596 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:09 smithi145 ceph-mon[21899]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:33:09.596 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:09 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:33:09.596 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:09 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:09.597 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:09 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:09.597 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:09 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:33:09.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:09 smithi172 ceph-mon[22747]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:33:09.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:09 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:33:09.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:09 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:09.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:09 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:09.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:09 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:33:10.563 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:33:10 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[75967]: ts=2024-04-22T15:33:10.511Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000153394s 2024-04-22T15:33:10.856 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:33:11.552 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:33:11.552 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 starting - - - - 2024-04-22T15:33:11.553 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (14m) 22s ago 21m 53.5M - 8.3.5 dad864ee21e9 20a9f6cd1859 2024-04-22T15:33:11.553 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 unknown 47s ago 21m 59.6M - 3.5 e1d6a67b021e 9704631ffa0e 2024-04-22T15:33:11.553 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443,9283,8765 running (78s) 22s ago 26m 451M - 19.0.0-3223-g5883d494 3d983c26e278 e7500ef71604 2024-04-22T15:33:11.553 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (11m) 47s ago 28m 501M - 19.0.0-3223-g5883d494 3d983c26e278 b2de97566cf4 2024-04-22T15:33:11.553 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (28m) 47s ago 28m 78.3M 2048M 17.2.0 e1d6a67b021e 28106a7bf16e 2024-04-22T15:33:11.553 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (27m) 22s ago 27m 59.7M 2048M 17.2.0 e1d6a67b021e a2fc3297953a 2024-04-22T15:33:11.553 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (27m) 47s ago 27m 56.5M 2048M 17.2.0 e1d6a67b021e b71d9b9f8509 2024-04-22T15:33:11.553 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (59s) 47s ago 23m 8028k - 1.5.0 0da6a335fe13 8c75c26fec05 2024-04-22T15:33:11.553 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (52s) 22s ago 22m 7759k - 1.5.0 0da6a335fe13 8fd2949e2eaf 2024-04-22T15:33:11.553 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (26m) 47s ago 26m 56.7M 2992M 17.2.0 e1d6a67b021e ea9cdc639916 2024-04-22T15:33:11.554 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (26m) 47s ago 26m 55.2M 2992M 17.2.0 e1d6a67b021e 3b9b37a8781f 2024-04-22T15:33:11.554 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (25m) 47s ago 25m 50.8M 2992M 17.2.0 e1d6a67b021e d9502dded8f8 2024-04-22T15:33:11.554 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (25m) 47s ago 25m 53.0M 2992M 17.2.0 e1d6a67b021e 8c72f30e54fc 2024-04-22T15:33:11.554 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (25m) 22s ago 25m 57.7M 3248M 17.2.0 e1d6a67b021e 80001f97ed35 2024-04-22T15:33:11.554 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (24m) 22s ago 24m 54.2M 3248M 17.2.0 e1d6a67b021e ec3ab1335b5c 2024-04-22T15:33:11.554 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (24m) 22s ago 24m 54.6M 3248M 17.2.0 e1d6a67b021e 45d475edf1ca 2024-04-22T15:33:11.554 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (23m) 22s ago 23m 52.8M 3248M 17.2.0 e1d6a67b021e cc9e1d9e7916 2024-04-22T15:33:11.554 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (26s) 22s ago 23m 46.0M - 2.43.0 a07b618ecd1d 51cf4df6e8d4 2024-04-22T15:33:11.554 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (21m) 47s ago 21m 77.9M - 17.2.0 e1d6a67b021e c90a636c947e 2024-04-22T15:33:11.554 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (21m) 22s ago 21m 75.5M - 17.2.0 e1d6a67b021e 20ac46e9bfda 2024-04-22T15:33:11.617 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:11 smithi145 ceph-mon[26606]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:33:11.617 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:11 smithi145 ceph-mon[21899]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:33:11.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:11 smithi172 ceph-mon[22747]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:33:11.960 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:33:11.960 INFO:teuthology.orchestra.run.smithi145.stdout: "mon": { 2024-04-22T15:33:11.960 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-22T15:33:11.960 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:33:11.960 INFO:teuthology.orchestra.run.smithi145.stdout: "mgr": { 2024-04-22T15:33:11.960 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:33:11.960 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:33:11.960 INFO:teuthology.orchestra.run.smithi145.stdout: "osd": { 2024-04-22T15:33:11.960 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T15:33:11.961 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:33:11.961 INFO:teuthology.orchestra.run.smithi145.stdout: "mds": {}, 2024-04-22T15:33:11.961 INFO:teuthology.orchestra.run.smithi145.stdout: "rgw": { 2024-04-22T15:33:11.961 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:33:11.961 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:33:11.961 INFO:teuthology.orchestra.run.smithi145.stdout: "overall": { 2024-04-22T15:33:11.961 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-22T15:33:11.961 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:33:11.961 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:33:11.961 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:33:12.305 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:33:12.306 INFO:teuthology.orchestra.run.smithi145.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:33:12.306 INFO:teuthology.orchestra.run.smithi145.stdout: "in_progress": true, 2024-04-22T15:33:12.306 INFO:teuthology.orchestra.run.smithi145.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-04-22T15:33:12.306 INFO:teuthology.orchestra.run.smithi145.stdout: "services_complete": [ 2024-04-22T15:33:12.306 INFO:teuthology.orchestra.run.smithi145.stdout: "mgr" 2024-04-22T15:33:12.306 INFO:teuthology.orchestra.run.smithi145.stdout: ], 2024-04-22T15:33:12.306 INFO:teuthology.orchestra.run.smithi145.stdout: "progress": "2/2 daemons upgraded", 2024-04-22T15:33:12.306 INFO:teuthology.orchestra.run.smithi145.stdout: "message": "Currently upgrading alertmanager daemons", 2024-04-22T15:33:12.306 INFO:teuthology.orchestra.run.smithi145.stdout: "is_paused": false 2024-04-22T15:33:12.306 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:33:12.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:12 smithi172 ceph-mon[22747]: from='client.25477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:33:12.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:12 smithi172 ceph-mon[22747]: from='client.25483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:33:12.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:12 smithi172 ceph-mon[22747]: from='client.15567 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:33:12.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:12 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2734584135' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:12.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:12 smithi145 ceph-mon[26606]: from='client.25477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:33:12.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:12 smithi145 ceph-mon[26606]: from='client.25483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:33:12.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:12 smithi145 ceph-mon[26606]: from='client.15567 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:33:12.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:12 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2734584135' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:12.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:12 smithi145 ceph-mon[21899]: from='client.25477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:33:12.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:12 smithi145 ceph-mon[21899]: from='client.25483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:33:12.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:12 smithi145 ceph-mon[21899]: from='client.15567 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:33:12.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:12 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2734584135' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:13.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:13 smithi145 ceph-mon[26606]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:33:13.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:13 smithi145 ceph-mon[26606]: from='client.25501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:33:13.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:13 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:13.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:13 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:13.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:13 smithi145 ceph-mon[21899]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:33:13.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:13 smithi145 ceph-mon[21899]: from='client.25501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:33:13.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:13 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:13.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:13 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:13.694 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:13 smithi172 ceph-mon[22747]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:33:13.694 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:13 smithi172 ceph-mon[22747]: from='client.25501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:33:13.694 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:13 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:13.694 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:13 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:15.256 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:15 smithi145 ceph-mon[26606]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:33:15.256 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:15 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:15.256 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:15 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:15.256 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:15 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:15.256 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:15 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:15.256 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:15 smithi145 ceph-mon[21899]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:33:15.256 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:15 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:15.256 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:15 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:15.256 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:15 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:15.257 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:15 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:15.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:15 smithi172 ceph-mon[22747]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:33:15.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:15 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:15.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:15 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:15.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:15 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:15.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:15 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:15.619 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:33:15] "GET /metrics HTTP/1.1" 200 37994 "" "Prometheus/2.43.0" 2024-04-22T15:33:17.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:17 smithi145 ceph-mon[21899]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:33:17.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:17 smithi145 ceph-mon[26606]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:33:17.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:17 smithi172 ceph-mon[22747]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:33:18.607 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:33:18 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[75967]: ts=2024-04-22T15:33:18.514Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003084729s 2024-04-22T15:33:19.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:18 smithi145 ceph-mon[21899]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:33:19.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:18 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:19.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:18 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:19.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:18 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:33:19.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:18 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:33:19.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:18 smithi145 ceph-mon[21899]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1004 B/s rd, 0 op/s 2024-04-22T15:33:19.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:18 smithi145 ceph-mon[21899]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:33:19.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:18 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:19.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:18 smithi145 ceph-mon[21899]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:33:19.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:18 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:33:19.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:18 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:33:19.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:18 smithi145 ceph-mon[21899]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:33:19.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:18 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:33:19.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:18 smithi145 ceph-mon[26606]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:33:19.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:18 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:19.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:18 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:19.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:18 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:33:19.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:18 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:33:19.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:18 smithi145 ceph-mon[26606]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1004 B/s rd, 0 op/s 2024-04-22T15:33:19.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:18 smithi145 ceph-mon[26606]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:33:19.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:18 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:19.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:18 smithi145 ceph-mon[26606]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:33:19.126 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:18 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:33:19.126 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:18 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:33:19.126 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:18 smithi145 ceph-mon[26606]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:33:19.126 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:18 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:33:19.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:18 smithi172 ceph-mon[22747]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:33:19.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:18 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:19.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:18 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:19.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:18 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:33:19.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:18 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:33:19.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:18 smithi172 ceph-mon[22747]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1004 B/s rd, 0 op/s 2024-04-22T15:33:19.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:18 smithi172 ceph-mon[22747]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:33:19.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:18 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:19.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:18 smithi172 ceph-mon[22747]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:33:19.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:18 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:33:19.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:18 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:33:19.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:18 smithi172 ceph-mon[22747]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:33:19.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:18 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:33:19.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:33:19.673+0000 7f42a94bd700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:33:19.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:33:19.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:33:19.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:33:19.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:33:19.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:19.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:19.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:33:19.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:33:19.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:33:19.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:33:19.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:33:19.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:33:19.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:33:19.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:33:19.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:33:19.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:33:19.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:33:19.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:33:19.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:19.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:19.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:33:19.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:33:19.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:33:19.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:33:19.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:33:19.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:33:19.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:33:19.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:33:19.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:33:19.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:33:19.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:33:19.879 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:33:19.879 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:33:19.879 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:33:19.879 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:33:19.879 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:33:19.879 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:33:19.879 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:33:19.879 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:33:19.879 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:33:19.879 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:33:19.879 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:33:19.880 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:19.880 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:19.881 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:33:19.881 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:33:19.881 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:33:19.881 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:33:19.881 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:33:19.881 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:33:19.881 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:33:19.881 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:33:19.882 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:33:19.882 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:33:19.882 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:33:19.882 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:33:19.882 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:19.882 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:19.883 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:33:19] ENGINE Bus STOPPING 2024-04-22T15:33:19.883 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:33:19] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-22T15:33:19.883 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:33:19] ENGINE Bus STOPPED 2024-04-22T15:33:19.883 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:33:19] ENGINE Bus STARTING 2024-04-22T15:33:19.883 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:33:19] ENGINE Serving on http://:::9283 2024-04-22T15:33:19.883 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:33:19] ENGINE Bus STARTED 2024-04-22T15:33:20.617 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:20 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:33:20.617 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:20 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T15:33:20.617 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:20 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi145:9093"}]: dispatch 2024-04-22T15:33:20.617 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:20 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:20.617 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:20 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:33:20.617 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:20 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:20.617 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:20 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:20.617 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:20 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:20.617 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:20 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:20.617 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:20 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:20.617 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:20 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:20.617 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:20 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:20.617 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:20 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:20.618 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:20 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:20.618 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:20 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:20.618 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:20 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:20.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:20 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:33:20.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:20 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T15:33:20.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:20 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi145:9093"}]: dispatch 2024-04-22T15:33:20.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:20 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:20.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:20 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:33:20.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:20 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:20.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:20 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:20.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:20 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:20.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:20 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:20.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:20 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:20.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:20 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:20.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:20 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:20.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:20 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:20.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:20 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:20.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:20 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:20.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:20 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:20.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:20 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:33:20.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:20 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T15:33:20.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:20 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi145:9093"}]: dispatch 2024-04-22T15:33:20.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:20 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:20.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:20 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:33:20.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:20 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:20.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:20 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:20.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:20 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:20.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:20 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:20.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:20 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:20.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:20 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:20.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:20 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:20.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:20 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:20.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:20 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:20.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:20 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:20.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:20 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:21.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:21 smithi172 ceph-mon[22747]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:33:21.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:33:21.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:33:21.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:33:21.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:33:21.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:21.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:21.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:33:21.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:33:21.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:33:21.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:33:21.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:33:21.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:33:21.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:33:21.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:33:21.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:33:21.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:33:21.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:33:21.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:33:21.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:21.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:21.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:33:21.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:33:21.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:33:21.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:33:21.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:33:21.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:33:21.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:33:21.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:33:21.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:33:21.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:33:21.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:33:21.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:33:21.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:33:21.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:33:21.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:33:21.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:33:21.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:33:21.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:33:21.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:33:21.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:33:21.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:33:21.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:33:21.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:21.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:21.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:33:21.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:33:21.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:33:21.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:33:21.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:33:21.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:33:21.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:33:21.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:33:21.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:33:21.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:33:21.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:33:21.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:33:21.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:21.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:21.472 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:21 smithi172 ceph-mon[22747]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T15:33:21.472 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:21 smithi172 ceph-mon[22747]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi145:9093"}]: dispatch 2024-04-22T15:33:21.472 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:21 smithi172 ceph-mon[22747]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-04-22T15:33:21.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:21 smithi145 ceph-mon[21899]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:33:21.566 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:33:21.566 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:33:21.566 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:33:21.566 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:33:21.566 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:21.566 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:21.566 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:33:21.567 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:33:21.567 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:33:21.567 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:33:21.567 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:33:21.567 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:33:21.567 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:33:21.567 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:33:21.567 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:33:21.567 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:33:21.567 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:33:21.567 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:33:21.567 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:21.567 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:21.567 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:33:21.568 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:33:21.568 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:33:21.568 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:33:21.568 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:33:21.568 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:33:21.568 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:33:21.568 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:33:21.568 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:33:21.568 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:33:21.568 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:33:21.568 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:33:21.568 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:33:21.568 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:33:21.568 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:33:21.568 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:33:21.568 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:33:21.569 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:33:21.569 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:33:21.569 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:33:21.569 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:33:21.569 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:33:21.569 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:21.569 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:21.569 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:33:21.569 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:33:21.569 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:33:21.569 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:33:21.569 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:33:21.569 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:33:21.569 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:33:21.570 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:33:21.570 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:33:21.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:21 smithi145 ceph-mon[26606]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:33:21.570 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:33:21.570 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:33:21.570 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:33:21.570 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:33:21.570 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:21.570 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:21.570 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:33:21.570 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:33:21.571 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:33:21.571 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:33:21.571 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:33:21.571 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:33:21.571 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:33:21.571 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:33:21.571 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:33:21.571 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:33:21.571 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:33:21.571 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:33:21.571 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:21.571 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:21.571 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:33:21.571 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:33:21.572 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:33:21.572 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:33:21.572 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:33:21.572 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:33:21.572 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:33:21.572 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:33:21.572 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:33:21.572 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:33:21.572 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:33:21.572 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:33:21.572 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:33:21.572 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:33:21.572 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:33:21.572 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:33:21.572 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:33:21.573 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:33:21.573 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:33:21.573 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:33:21.573 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:33:21.573 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:33:21.573 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:21.573 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:21.573 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:33:21.573 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:33:21.573 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:33:21.573 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:33:21.573 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:33:21.573 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:33:21.573 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:33:21.573 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:33:21.573 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:33:21.574 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:33:21.574 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:33:21.574 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:33:21.574 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:21.574 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:21.574 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:21 smithi145 ceph-mon[21899]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T15:33:21.574 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:21 smithi145 ceph-mon[21899]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi145:9093"}]: dispatch 2024-04-22T15:33:21.574 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:21 smithi145 ceph-mon[21899]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-04-22T15:33:21.575 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:33:21.575 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:33:21.575 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:33:21.575 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:21.575 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:21.575 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:21 smithi145 ceph-mon[26606]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T15:33:21.575 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:21 smithi145 ceph-mon[26606]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi145:9093"}]: dispatch 2024-04-22T15:33:21.575 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:21 smithi145 ceph-mon[26606]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-04-22T15:33:22.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:22 smithi145 ceph-mon[21899]: Upgrade: Updating grafana.a 2024-04-22T15:33:22.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:22 smithi145 ceph-mon[21899]: Deploying daemon grafana.a on smithi172 2024-04-22T15:33:22.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:22 smithi145 ceph-mon[26606]: Upgrade: Updating grafana.a 2024-04-22T15:33:22.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:22 smithi145 ceph-mon[26606]: Deploying daemon grafana.a on smithi172 2024-04-22T15:33:22.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:22 smithi172 ceph-mon[22747]: Upgrade: Updating grafana.a 2024-04-22T15:33:22.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:22 smithi172 ceph-mon[22747]: Deploying daemon grafana.a on smithi172 2024-04-22T15:33:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:23 smithi145 ceph-mon[21899]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:33:23.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:23 smithi145 ceph-mon[26606]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:33:23.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:23 smithi172 ceph-mon[22747]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:33:25.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:25 smithi145 ceph-mon[21899]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-04-22T15:33:25.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:25 smithi145 ceph-mon[26606]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-04-22T15:33:25.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:33:25] "GET /metrics HTTP/1.1" 200 37994 "" "Prometheus/2.43.0" 2024-04-22T15:33:25.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:25 smithi172 ceph-mon[22747]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 625 B/s rd, 0 op/s 2024-04-22T15:33:27.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:27 smithi145 ceph-mon[21899]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:33:27.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:27 smithi145 ceph-mon[26606]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:33:27.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:27 smithi172 ceph-mon[22747]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:33:29.437 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:29 smithi172 ceph-mon[22747]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:33:29.437 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:29 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:33:29.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:29 smithi145 ceph-mon[26606]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:33:29.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:29 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:33:29.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:29 smithi145 ceph-mon[21899]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:33:29.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:29 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:33:31.300 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:31 smithi172 systemd[1]: Stopping Ceph grafana.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:33:31.300 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:31 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[46687]: t=2024-04-22T15:33:31+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-04-22T15:33:31.599 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:31 smithi172 ceph-mon[22747]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:33:31.600 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:31 smithi172 podman[63415]: 2024-04-22 15:33:31.297567091 +0000 UTC m=+0.108481926 container died 20a9f6cd18595d1d691a98eefc0be0d47481ceb7fee46ebeaad70c648d121c6f (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a, build-date=2022-03-28T10:36:18.413762, io.openshift.expose-services=, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, architecture=x86_64, release=236.1648460182, version=8.5, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, io.buildah.version=1.24.2, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-22T15:33:31.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:31 smithi145 ceph-mon[21899]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:33:31.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:31 smithi145 ceph-mon[26606]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:33:31.932 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:31 smithi172 podman[63415]: 2024-04-22 15:33:31.670442203 +0000 UTC m=+0.481357045 container cleanup 20a9f6cd18595d1d691a98eefc0be0d47481ceb7fee46ebeaad70c648d121c6f (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vendor=Red Hat, Inc., maintainer=Paul Cuzner , io.openshift.tags=base rhel8, distribution-scope=public, release=236.1648460182, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.24.2, build-date=2022-03-28T10:36:18.413762, vcs-type=git) 2024-04-22T15:33:31.933 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:31 smithi172 bash[63415]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a 2024-04-22T15:33:32.218 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:31 smithi172 podman[63429]: 2024-04-22 15:33:31.929998387 +0000 UTC m=+0.633993133 container remove 20a9f6cd18595d1d691a98eefc0be0d47481ceb7fee46ebeaad70c648d121c6f (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, build-date=2022-03-28T10:36:18.413762, release=236.1648460182, maintainer=Paul Cuzner , distribution-scope=public, architecture=x86_64, description=Ceph Grafana Container, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.24.2, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=) 2024-04-22T15:33:32.218 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@grafana.a.service: Deactivated successfully. 2024-04-22T15:33:32.218 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 systemd[1]: Stopped Ceph grafana.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:33:32.218 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@grafana.a.service: Consumed 5.608s CPU time. 2024-04-22T15:33:32.642 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 systemd[1]: Starting Ceph grafana.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:33:32.642 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 podman[63560]: 2024-04-22 15:33:32.514527255 +0000 UTC m=+0.106757103 container create 2f825d15ab1081a1ab5ec9311e4549900b86622e6f3868cfc7bdf62250a400fc (image=quay.io/ceph/grafana:9.4.12, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-04-22T15:33:32.642 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 podman[63560]: 2024-04-22 15:33:32.426307756 +0000 UTC m=+0.018537616 image pull 2bacad6d85d802cd176c146528adcc0e47a8a83e94bc4a8b7ba1a9b9ef2c65e7 quay.io/ceph/grafana:9.4.12 2024-04-22T15:33:32.902 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 podman[63560]: 2024-04-22 15:33:32.733668711 +0000 UTC m=+0.325898569 container init 2f825d15ab1081a1ab5ec9311e4549900b86622e6f3868cfc7bdf62250a400fc (image=quay.io/ceph/grafana:9.4.12, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-04-22T15:33:32.902 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 podman[63560]: 2024-04-22 15:33:32.741405274 +0000 UTC m=+0.333635125 container start 2f825d15ab1081a1ab5ec9311e4549900b86622e6f3868cfc7bdf62250a400fc (image=quay.io/ceph/grafana:9.4.12, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-04-22T15:33:32.902 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 bash[63560]: 2f825d15ab1081a1ab5ec9311e4549900b86622e6f3868cfc7bdf62250a400fc 2024-04-22T15:33:32.902 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 systemd[1]: Started Ceph grafana.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:33:32.902 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=settings t=2024-04-22T15:33:32.888326686Z level=info msg="Starting Grafana" version=9.4.12 commit=37ee25ba46 branch=HEAD compiled=2023-05-22T15:58:05Z 2024-04-22T15:33:32.903 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=settings t=2024-04-22T15:33:32.888488692Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-04-22T15:33:32.903 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=settings t=2024-04-22T15:33:32.888498837Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-04-22T15:33:32.903 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=settings t=2024-04-22T15:33:32.888502011Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-04-22T15:33:32.903 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=settings t=2024-04-22T15:33:32.888504603Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana" 2024-04-22T15:33:32.903 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=settings t=2024-04-22T15:33:32.888507128Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana" 2024-04-22T15:33:32.903 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=settings t=2024-04-22T15:33:32.888509471Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins" 2024-04-22T15:33:32.903 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=settings t=2024-04-22T15:33:32.888511949Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning" 2024-04-22T15:33:32.903 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=settings t=2024-04-22T15:33:32.888515081Z level=info msg="Config overridden from command line" arg="default.log.mode=console" 2024-04-22T15:33:32.903 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=settings t=2024-04-22T15:33:32.888517681Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-04-22T15:33:32.903 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=settings t=2024-04-22T15:33:32.888520096Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-04-22T15:33:32.903 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=settings t=2024-04-22T15:33:32.888522722Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-04-22T15:33:32.903 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=settings t=2024-04-22T15:33:32.888525195Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-04-22T15:33:32.903 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=settings t=2024-04-22T15:33:32.888527672Z level=info msg="Path Home" path=/usr/share/grafana 2024-04-22T15:33:32.903 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=settings t=2024-04-22T15:33:32.888530869Z level=info msg="Path Data" path=/var/lib/grafana 2024-04-22T15:33:32.903 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=settings t=2024-04-22T15:33:32.888533151Z level=info msg="Path Logs" path=/var/log/grafana 2024-04-22T15:33:32.904 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=settings t=2024-04-22T15:33:32.888535318Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-04-22T15:33:32.904 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=settings t=2024-04-22T15:33:32.888537802Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-04-22T15:33:32.904 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=settings t=2024-04-22T15:33:32.888540174Z level=info msg="App mode production" 2024-04-22T15:33:32.904 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=sqlstore t=2024-04-22T15:33:32.888599665Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-04-22T15:33:32.904 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=sqlstore t=2024-04-22T15:33:32.888610656Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-04-22T15:33:32.904 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.899626855Z level=info msg="Starting DB migrations" 2024-04-22T15:33:33.218 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.904705128Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-04-22T15:33:33.218 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.916006319Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-04-22T15:33:33.219 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.917124534Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-04-22T15:33:33.219 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.917978197Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-04-22T15:33:33.219 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.919233867Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-04-22T15:33:33.219 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.920371867Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-04-22T15:33:33.219 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.921446401Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-04-22T15:33:33.219 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.92233455Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-04-22T15:33:33.219 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.923174914Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-04-22T15:33:33.219 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.923760963Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-04-22T15:33:33.219 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.924508852Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-04-22T15:33:33.219 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.925788865Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-04-22T15:33:33.219 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.926160194Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-04-22T15:33:33.219 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.92734389Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-04-22T15:33:33.219 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.928692881Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-04-22T15:33:33.219 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.929792811Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-04-22T15:33:33.220 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.930202517Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-04-22T15:33:33.220 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.931268775Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-04-22T15:33:33.220 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.932352875Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-04-22T15:33:33.220 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.932644637Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-04-22T15:33:33.220 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.933733295Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-04-22T15:33:33.221 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.93479182Z level=info msg="Executing migration" id="create provenance_type table" 2024-04-22T15:33:33.221 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.935247306Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-04-22T15:33:33.221 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.936083231Z level=info msg="Executing migration" id="create alert_image table" 2024-04-22T15:33:33.221 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.936929334Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-04-22T15:33:33.221 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.937354087Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-04-22T15:33:33.221 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.937615439Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-04-22T15:33:33.221 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.93801702Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-04-22T15:33:33.221 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.938447739Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-04-22T15:33:33.221 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.938519745Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-04-22T15:33:33.221 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.938752774Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-04-22T15:33:33.221 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.93922687Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-04-22T15:33:33.221 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.939666625Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-04-22T15:33:33.221 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.939938694Z level=info msg="Executing migration" id="create secrets table" 2024-04-22T15:33:33.221 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.940342027Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-04-22T15:33:33.221 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.946589752Z level=info msg="Executing migration" id="add name column into data_keys" 2024-04-22T15:33:33.222 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.947790716Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-04-22T15:33:33.222 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.948133461Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-04-22T15:33:33.222 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.95450108Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-04-22T15:33:33.222 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.960796893Z level=info msg="Executing migration" id="add column hidden to role table" 2024-04-22T15:33:33.222 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.961977542Z level=info msg="Executing migration" id="create query_history table v1" 2024-04-22T15:33:33.222 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.962465877Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-04-22T15:33:33.222 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.962942483Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-04-22T15:33:33.222 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.96325562Z level=info msg="Executing migration" id="teams permissions migration" 2024-04-22T15:33:33.222 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.96366677Z level=info msg="Executing migration" id="dashboard permissions" 2024-04-22T15:33:33.222 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.964966987Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-04-22T15:33:33.222 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.965659313Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-04-22T15:33:33.222 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.965964406Z level=info msg="Executing migration" id="alerting notification permissions" 2024-04-22T15:33:33.222 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.966334611Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-04-22T15:33:33.222 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.966737572Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-04-22T15:33:33.223 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.967201404Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-04-22T15:33:33.223 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.968324056Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-04-22T15:33:33.223 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.968639663Z level=info msg="Executing migration" id="create correlation table v1" 2024-04-22T15:33:33.223 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.96909317Z level=info msg="Executing migration" id="add index correlations.uid" 2024-04-22T15:33:33.223 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.96955998Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-04-22T15:33:33.223 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.970010391Z level=info msg="Executing migration" id="add correlation config column" 2024-04-22T15:33:33.223 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.971170362Z level=info msg="Executing migration" id="create entity_events table" 2024-04-22T15:33:33.223 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.97157194Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-04-22T15:33:33.223 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.971997548Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-22T15:33:33.223 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.972102688Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-22T15:33:33.223 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.972396818Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-22T15:33:33.223 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.972493198Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-22T15:33:33.223 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.972790926Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-04-22T15:33:33.223 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.973256192Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-04-22T15:33:33.223 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.974103798Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-04-22T15:33:33.224 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.974595567Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-22T15:33:33.224 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.975492018Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-04-22T15:33:33.224 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.976356189Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-22T15:33:33.224 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.977281344Z level=info msg="Executing migration" id="Drop public config table" 2024-04-22T15:33:33.224 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.977738442Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-04-22T15:33:33.224 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.980493293Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-04-22T15:33:33.224 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.983443884Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-22T15:33:33.224 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.986358774Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-04-22T15:33:33.224 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.986836197Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-04-22T15:33:33.225 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.990748897Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-04-22T15:33:33.225 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.9919288Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-04-22T15:33:33.225 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.993098963Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-04-22T15:33:33.225 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.993446379Z level=info msg="Executing migration" id="add share column" 2024-04-22T15:33:33.225 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.994702174Z level=info msg="Executing migration" id="create default alerting folders" 2024-04-22T15:33:33.225 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.995571826Z level=info msg="Executing migration" id="create file table" 2024-04-22T15:33:33.225 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:32 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:32.998415543Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-04-22T15:33:33.225 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:33.001078374Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-04-22T15:33:33.225 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:33.003813454Z level=info msg="Executing migration" id="create file_meta table" 2024-04-22T15:33:33.225 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:33.006592282Z level=info msg="Executing migration" id="file table idx: path key" 2024-04-22T15:33:33.225 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:33.009367775Z level=info msg="Executing migration" id="set path collation in file table" 2024-04-22T15:33:33.225 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:33.009692645Z level=info msg="Executing migration" id="managed permissions migration" 2024-04-22T15:33:33.225 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:33.012572594Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-04-22T15:33:33.226 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:33.015338718Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-04-22T15:33:33.226 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:33.018148006Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-04-22T15:33:33.226 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:33.019470114Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-04-22T15:33:33.226 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:33.02233696Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-04-22T15:33:33.226 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:33.025086417Z level=info msg="Executing migration" id="update group index for alert rules" 2024-04-22T15:33:33.226 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:33.027996864Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-04-22T15:33:33.226 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:33.028565323Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-04-22T15:33:33.226 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:33.031402653Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-04-22T15:33:33.226 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:33.032746744Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-04-22T15:33:33.226 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:33.034167002Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-04-22T15:33:33.226 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:33.036985552Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-04-22T15:33:33.226 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:33.050445806Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-04-22T15:33:33.226 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:33.053006618Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-04-22T15:33:33.227 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:33.055488946Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-04-22T15:33:33.227 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:33.060033494Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-04-22T15:33:33.227 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:33.060912185Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-04-22T15:33:33.227 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:33.063375961Z level=info msg="Executing migration" id="create folder table" 2024-04-22T15:33:33.227 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:33.065959473Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-04-22T15:33:33.227 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:33.068602441Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-04-22T15:33:33.227 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:33.069297951Z level=info msg="Executing migration" id="Update folder title length" 2024-04-22T15:33:33.227 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:33.071927309Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-04-22T15:33:33.227 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=migrator t=2024-04-22T15:33:33.074647747Z level=info msg="migrations completed" performed=105 skipped=376 duration=169.967664ms 2024-04-22T15:33:33.227 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=sqlstore t=2024-04-22T15:33:33.075177715Z level=info msg="Created default organization" 2024-04-22T15:33:33.227 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=plugin.finder t=2024-04-22T15:33:33.110452654Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled 2024-04-22T15:33:33.227 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=secrets t=2024-04-22T15:33:33.110545455Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-04-22T15:33:33.227 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=query_data t=2024-04-22T15:33:33.114509698Z level=info msg="Query Service initialization" 2024-04-22T15:33:33.227 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=live.push_http t=2024-04-22T15:33:33.117205736Z level=info msg="Live Push Gateway initialization" 2024-04-22T15:33:33.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:33 smithi145 ceph-mon[21899]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:33:33.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:33 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:33.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:33 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:33.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:33 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:33:33.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:33 smithi145 ceph-mon[26606]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:33:33.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:33 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:33.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:33 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:33.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:33 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:33:33.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-mon[22747]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:33:33.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:33.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:33.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:33:34.218 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=infra.usagestats.collector t=2024-04-22T15:33:33.790560816Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-04-22T15:33:34.218 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=provisioning.datasources t=2024-04-22T15:33:33.794039276Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-04-22T15:33:34.218 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=provisioning.datasources t=2024-04-22T15:33:33.794054262Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-04-22T15:33:34.218 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=provisioning.datasources t=2024-04-22T15:33:33.794156692Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-04-22T15:33:34.219 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=provisioning.alerting t=2024-04-22T15:33:33.816399964Z level=info msg="starting to provision alerting" 2024-04-22T15:33:34.219 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=provisioning.alerting t=2024-04-22T15:33:33.816415814Z level=info msg="finished to provision alerting" 2024-04-22T15:33:34.219 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=grafanaStorageLogger t=2024-04-22T15:33:33.816548581Z level=info msg="storage starting" 2024-04-22T15:33:34.219 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=ngalert.state.manager t=2024-04-22T15:33:33.816637531Z level=info msg="Warming state cache for startup" 2024-04-22T15:33:34.219 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=http.server t=2024-04-22T15:33:33.819199056Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-04-22T15:33:34.219 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=ngalert.state.manager t=2024-04-22T15:33:33.828633238Z level=info msg="State cache has been initialized" states=0 duration=11.994779ms 2024-04-22T15:33:34.219 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=ticker t=2024-04-22T15:33:33.828696705Z level=info msg=starting first_tick=2024-04-22T15:33:40Z 2024-04-22T15:33:34.219 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=ngalert.multiorg.alertmanager t=2024-04-22T15:33:33.828710023Z level=info msg="starting MultiOrg Alertmanager" 2024-04-22T15:33:34.219 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=sqlstore.transactions t=2024-04-22T15:33:33.841326853Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-22T15:33:34.219 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=sqlstore.transactions t=2024-04-22T15:33:33.852886455Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-04-22T15:33:34.219 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:33:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=sqlstore.transactions t=2024-04-22T15:33:33.863941354Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=2 code="database is locked" 2024-04-22T15:33:35.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:35 smithi145 ceph-mon[21899]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:33:35.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:35 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:33:35.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:35 smithi145 ceph-mon[26606]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:33:35.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:35 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:33:35.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:33:35] "GET /metrics HTTP/1.1" 200 37991 "" "Prometheus/2.43.0" 2024-04-22T15:33:35.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:35 smithi172 ceph-mon[22747]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:33:35.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:35 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:33:37.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:37 smithi145 ceph-mon[21899]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:33:37.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:37 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:37.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:37 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:37 smithi145 ceph-mon[26606]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:33:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:37 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:37 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:37.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:37 smithi172 ceph-mon[22747]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:33:37.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:37 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:37.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:37 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:39.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:39 smithi172 ceph-mon[22747]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:33:39.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:39 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:39.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:39 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:39.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:39 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:33:39.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:39 smithi145 ceph-mon[21899]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:33:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:39 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:39 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:39 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:33:39.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:39 smithi145 ceph-mon[26606]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:33:39.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:39 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:39.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:39 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:39.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:39 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:33:41.586 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:41 smithi172 ceph-mon[22747]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:33:41.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:41 smithi145 ceph-mon[21899]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:33:41.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:41 smithi145 ceph-mon[26606]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:33:42.667 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:33:42.667 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:42 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:42.667 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:42 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:42.667 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:42 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:33:42.667 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:42 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:33:42.667 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:42 smithi145 ceph-mon[21899]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:33:42.667 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:42 smithi145 ceph-mon[21899]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:33:42.667 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:42 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:42.667 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:42 smithi145 ceph-mon[21899]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:33:42.668 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:42 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:33:42.668 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:42 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:33:42.668 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:42 smithi145 ceph-mon[21899]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:33:42.668 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:42 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:42.668 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:42 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:42.668 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:42 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:33:42.668 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:42 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:33:42.668 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:42 smithi145 ceph-mon[26606]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:33:42.668 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:42 smithi145 ceph-mon[26606]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:33:42.668 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:42 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:42.668 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:42 smithi145 ceph-mon[26606]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:33:42.668 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:42 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:33:42.668 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:42 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:33:42.668 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:42 smithi145 ceph-mon[26606]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:33:42.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:42 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:42.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:42 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:42.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:42 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:33:42.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:42 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:33:42.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:42 smithi172 ceph-mon[22747]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:33:42.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:42 smithi172 ceph-mon[22747]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:33:42.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:42 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:42.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:42 smithi172 ceph-mon[22747]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:33:42.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:42 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:33:42.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:42 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:33:42.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:42 smithi172 ceph-mon[22747]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:33:43.328 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:33:43.328 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (35s) 30s ago 22m 16.0M - 0.25.0 c8568f914cd2 1c58ccf37ffd 2024-04-22T15:33:43.328 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (10s) 6s ago 22m 51.3M - 9.4.12 2bacad6d85d8 2f825d15ab10 2024-04-22T15:33:43.328 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 unknown 30s ago 21m 59.6M - 3.5 e1d6a67b021e 9704631ffa0e 2024-04-22T15:33:43.328 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443,9283,8765 running (110s) 6s ago 27m 452M - 19.0.0-3223-g5883d494 3d983c26e278 e7500ef71604 2024-04-22T15:33:43.328 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (11m) 30s ago 29m 505M - 19.0.0-3223-g5883d494 3d983c26e278 b2de97566cf4 2024-04-22T15:33:43.328 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (29m) 30s ago 29m 79.5M 2048M 17.2.0 e1d6a67b021e 28106a7bf16e 2024-04-22T15:33:43.328 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (27m) 6s ago 27m 59.7M 2048M 17.2.0 e1d6a67b021e a2fc3297953a 2024-04-22T15:33:43.328 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (27m) 30s ago 27m 56.7M 2048M 17.2.0 e1d6a67b021e b71d9b9f8509 2024-04-22T15:33:43.329 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (90s) 30s ago 23m 8082k - 1.5.0 0da6a335fe13 8c75c26fec05 2024-04-22T15:33:43.329 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (84s) 6s ago 23m 16.0M - 1.5.0 0da6a335fe13 8fd2949e2eaf 2024-04-22T15:33:43.329 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (27m) 30s ago 27m 56.7M 2992M 17.2.0 e1d6a67b021e ea9cdc639916 2024-04-22T15:33:43.329 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (26m) 30s ago 26m 55.2M 2992M 17.2.0 e1d6a67b021e 3b9b37a8781f 2024-04-22T15:33:43.329 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (26m) 30s ago 26m 50.8M 2992M 17.2.0 e1d6a67b021e d9502dded8f8 2024-04-22T15:33:43.329 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (25m) 30s ago 25m 53.0M 2992M 17.2.0 e1d6a67b021e 8c72f30e54fc 2024-04-22T15:33:43.329 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (25m) 6s ago 25m 57.7M 3248M 17.2.0 e1d6a67b021e 80001f97ed35 2024-04-22T15:33:43.329 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (25m) 6s ago 25m 54.2M 3248M 17.2.0 e1d6a67b021e ec3ab1335b5c 2024-04-22T15:33:43.329 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (24m) 6s ago 24m 54.6M 3248M 17.2.0 e1d6a67b021e 45d475edf1ca 2024-04-22T15:33:43.329 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (24m) 6s ago 24m 52.8M 3248M 17.2.0 e1d6a67b021e cc9e1d9e7916 2024-04-22T15:33:43.329 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (58s) 6s ago 23m 55.8M - 2.43.0 a07b618ecd1d 51cf4df6e8d4 2024-04-22T15:33:43.329 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (22m) 30s ago 22m 78.0M - 17.2.0 e1d6a67b021e c90a636c947e 2024-04-22T15:33:43.329 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (22m) 6s ago 22m 75.7M - 17.2.0 e1d6a67b021e 20ac46e9bfda 2024-04-22T15:33:43.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:33:43.495+0000 7f42a94bd700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:33:43.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:33:43.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:33:43.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:33:43.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:33:43.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:43.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:43.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:33:43.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:33:43.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:33:43.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:33:43.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:33:43.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:33:43.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:33:43.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:33:43.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:33:43.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:33:43.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:33:43.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:33:43.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:43.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:43.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:33:43.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:33:43.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:33:43.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:33:43.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:33:43.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:33:43.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:33:43.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:33:43.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:33:43.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:33:43.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:33:43.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:33:43.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:33:43.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:33:43.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:33:43.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:33:43.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:33:43.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:33:43.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:33:43.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:33:43.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:33:43.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:33:43.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:43.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:43.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:33:43.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:33:43.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:33:43.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:33:43.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:33:43.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:33:43.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:33:43.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:33:43.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:33:43.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:33:43.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:33:43.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:33:43.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:43.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:43.798 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:33:43.798 INFO:teuthology.orchestra.run.smithi145.stdout: "mon": { 2024-04-22T15:33:43.798 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-22T15:33:43.798 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:33:43.798 INFO:teuthology.orchestra.run.smithi145.stdout: "mgr": { 2024-04-22T15:33:43.798 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:33:43.798 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:33:43.798 INFO:teuthology.orchestra.run.smithi145.stdout: "osd": { 2024-04-22T15:33:43.798 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T15:33:43.798 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:33:43.798 INFO:teuthology.orchestra.run.smithi145.stdout: "mds": {}, 2024-04-22T15:33:43.798 INFO:teuthology.orchestra.run.smithi145.stdout: "rgw": { 2024-04-22T15:33:43.799 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:33:43.799 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:33:43.799 INFO:teuthology.orchestra.run.smithi145.stdout: "overall": { 2024-04-22T15:33:43.799 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-22T15:33:43.799 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:33:43.799 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:33:43.799 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:33:43.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='client.25507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:33:43.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='client.25513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:33:43.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='client.15597 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:33:43.969 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:33:43.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:33:43.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:33:43.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:33:43.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:33:43.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:43.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:43.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:33:43.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:33:43.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:33:43.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:33:43.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:33:43.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:33:43.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:33:43.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:33:43.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:33:43.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:33:43.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:33:43.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:33:43.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:43.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:43.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:33:43.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:33:43.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:33:43.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:33:43.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:33:43.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:33:43.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:33:43.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:33:43.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:33:43.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:33:43.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:33:43.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:33:43.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:33:43.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:33:43.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:33:43.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:33:43.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:33:43.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:33:43.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:33:43.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:33:43.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:33:43.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:33:43.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:43.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:43.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:33:43.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:33:43.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:33:43.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:33:43.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:33:43.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:33:43.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:33:43.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:33:43.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:33:43.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:33:43.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:33:43.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:33:43.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:43.973 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:43.973 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T15:33:43.973 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T15:33:43.973 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:33:43.973 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:43.973 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:43.973 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:43.973 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:43.973 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:43.973 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:43.973 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:43.973 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:43.973 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:43.973 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:43.973 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:43.974 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:43.974 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:43.974 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:43.974 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: Upgrade: Finalizing container_image settings 2024-04-22T15:33:43.974 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:43.974 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T15:33:43.974 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T15:33:43.974 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:33:43.974 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T15:33:43.974 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T15:33:43.974 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T15:33:43.974 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T15:33:43.974 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T15:33:43.975 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T15:33:43.975 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T15:33:43.975 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T15:33:43.975 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T15:33:43.975 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:33:43.975 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T15:33:43.975 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T15:33:43.975 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T15:33:43.975 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T15:33:43.975 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T15:33:43.975 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T15:33:43.975 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T15:33:43.975 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:33:43.975 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:33:43.975 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:33:43.975 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:33:43.976 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:33:43.976 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:33:43.976 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T15:33:43.976 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T15:33:43.976 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:33:43.976 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:33:43.976 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:33:43.976 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:43 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:44.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='client.25507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:33:44.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='client.25513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:33:44.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='client.15597 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:33:44.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:33:44.124 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:33:44.124 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:33:44.124 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:33:44.124 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:33:44.125 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:44.125 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:44.125 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:33:44.125 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:33:44.125 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:33:44.126 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:33:44.126 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:33:44.126 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:33:44.126 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:33:44.126 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:33:44.126 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:33:44.127 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:33:44.127 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:33:44.127 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:33:44.127 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:44.127 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:44.127 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:33:44.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:33:44.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:33:44.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:33:44.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:33:44.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:33:44.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:33:44.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:33:44.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:33:44.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:33:44.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:33:44.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:33:44.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:33:44.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:33:44.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:33:44.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:33:44.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:33:44.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:33:44.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:33:44.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:33:44.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:33:44.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:33:44.132 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:44.132 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:44.132 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:33:44.132 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:33:44.132 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:33:44.132 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:33:44.133 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:33:44.133 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:33:44.133 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:33:44.134 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='client.25507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:33:44.134 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='client.25513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:33:44.134 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='client.15597 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:33:44.135 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:33:44.135 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:33:44.135 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:33:44.135 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:33:44.135 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:33:44.135 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:44.136 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:44.136 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:33:44.136 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:33:44.136 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:33:44.136 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:33:44.137 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:33:44.137 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:33:44.137 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:33:44.137 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:33:44.137 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:33:44.137 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:33:44.138 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:33:44.138 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:33:44.138 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:44.139 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:44.139 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:33:44.139 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:33:44.139 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:33:44.139 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:33:44.139 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:33:44.140 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:33:44.140 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:33:44.140 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:33:44.140 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:33:44.140 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:33:44.141 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:33:44.141 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:33:44.141 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:33:44.141 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:33:44.141 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:33:44.142 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:33:44.142 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:33:44.142 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:33:44.142 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:33:44.142 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:33:44.142 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:33:44.143 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:33:44.143 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:44.143 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:44.143 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:33:44.143 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:33:44.144 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:33:44.144 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:33:44.144 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:33:44.144 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:33:44.144 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:33:44.145 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:33:44.145 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:33:44.146 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:33:44.146 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:33:44.146 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:33:44.146 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:44.146 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:44.147 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T15:33:44.147 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T15:33:44.147 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:33:44.147 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:44.147 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:44.147 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:44.148 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:44.148 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:44.148 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:44.148 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:44.148 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:44.149 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:44.149 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:44.149 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:44.149 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:44.149 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:44.150 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:44.150 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: Upgrade: Finalizing container_image settings 2024-04-22T15:33:44.150 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:44.150 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T15:33:44.150 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T15:33:44.150 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:33:44.151 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T15:33:44.151 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T15:33:44.151 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T15:33:44.151 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T15:33:44.151 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T15:33:44.152 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T15:33:44.152 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T15:33:44.152 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T15:33:44.152 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T15:33:44.152 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:33:44.152 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T15:33:44.153 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T15:33:44.153 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T15:33:44.153 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T15:33:44.153 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T15:33:44.153 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T15:33:44.153 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T15:33:44.154 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:33:44.154 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:33:44.154 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:33:44.155 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:33:44.155 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:33:44.155 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:33:44.155 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:44.155 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:44.155 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T15:33:44.155 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T15:33:44.155 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:33:44.155 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:44.155 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:44.155 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:44.155 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:44.156 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:44.156 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:44.156 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:44.156 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:44.156 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:44.156 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:44.156 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:44.156 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:44.156 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:44.156 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:44.156 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: Upgrade: Finalizing container_image settings 2024-04-22T15:33:44.157 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:33:44.157 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:33:44.157 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:33:44.157 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:33:44.157 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:33:44.157 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T15:33:44.157 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T15:33:44.157 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:33:44.157 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:33:44.157 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:33:44.157 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:44.158 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:44.158 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T15:33:44.158 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T15:33:44.158 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:33:44.158 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T15:33:44.158 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T15:33:44.158 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T15:33:44.158 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T15:33:44.158 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T15:33:44.158 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T15:33:44.158 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T15:33:44.158 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T15:33:44.158 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T15:33:44.159 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:33:44.159 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T15:33:44.159 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T15:33:44.159 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T15:33:44.159 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T15:33:44.159 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T15:33:44.159 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T15:33:44.159 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T15:33:44.159 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:33:44.159 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:33:44.159 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:33:44.159 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:33:44.159 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:33:44.159 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:33:44.160 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T15:33:44.160 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T15:33:44.160 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:33:44.160 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:33:44.160 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:33:44.160 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:43 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:44.165 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:33:44.165 INFO:teuthology.orchestra.run.smithi145.stdout: "target_image": null, 2024-04-22T15:33:44.166 INFO:teuthology.orchestra.run.smithi145.stdout: "in_progress": false, 2024-04-22T15:33:44.166 INFO:teuthology.orchestra.run.smithi145.stdout: "which": "", 2024-04-22T15:33:44.166 INFO:teuthology.orchestra.run.smithi145.stdout: "services_complete": [], 2024-04-22T15:33:44.166 INFO:teuthology.orchestra.run.smithi145.stdout: "progress": null, 2024-04-22T15:33:44.166 INFO:teuthology.orchestra.run.smithi145.stdout: "message": "", 2024-04-22T15:33:44.166 INFO:teuthology.orchestra.run.smithi145.stdout: "is_paused": false 2024-04-22T15:33:44.166 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:33:44.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:44 smithi145 ceph-mon[26606]: Upgrade: Complete! 2024-04-22T15:33:44.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:44 smithi145 ceph-mon[26606]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:33:44.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:44 smithi145 ceph-mon[26606]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 664 B/s rd, 0 op/s 2024-04-22T15:33:44.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:44 smithi145 ceph-mon[26606]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:33:44.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:44 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:33:44.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:44 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:33:44.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:44 smithi145 ceph-mon[26606]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:33:44.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:44 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/4267169174' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:44.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:44 smithi145 ceph-mon[26606]: from='client.25531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:33:44.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:44 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:44.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:44 smithi145 ceph-mon[21899]: Upgrade: Complete! 2024-04-22T15:33:44.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:44 smithi145 ceph-mon[21899]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:33:44.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:44 smithi145 ceph-mon[21899]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 664 B/s rd, 0 op/s 2024-04-22T15:33:44.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:44 smithi145 ceph-mon[21899]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:33:44.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:44 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:33:44.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:44 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:33:44.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:44 smithi145 ceph-mon[21899]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:33:44.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:44 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/4267169174' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:44.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:44 smithi145 ceph-mon[21899]: from='client.25531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:33:44.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:44 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:44.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:44 smithi172 ceph-mon[22747]: Upgrade: Complete! 2024-04-22T15:33:44.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:44 smithi172 ceph-mon[22747]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:33:44.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:44 smithi172 ceph-mon[22747]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 664 B/s rd, 0 op/s 2024-04-22T15:33:44.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:44 smithi172 ceph-mon[22747]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:33:44.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:44 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:33:44.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:44 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:33:44.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:44 smithi172 ceph-mon[22747]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:33:44.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:44 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/4267169174' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:33:44.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:44 smithi172 ceph-mon[22747]: from='client.25531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:33:44.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:44 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:45.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:33:45] "GET /metrics HTTP/1.1" 200 37991 "" "Prometheus/2.43.0" 2024-04-22T15:33:45.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:33:45.437+0000 7f42a94bd700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:33:45.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:33:45.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:33:45.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:33:45.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:33:45.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:45.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:45.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:33:45.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:33:45.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:33:45.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:33:45.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:33:45.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:33:45.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:33:45.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:33:45.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:33:45.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:33:45.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:33:45.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:33:45.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:45.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:45.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:33:45.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:33:45.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:33:45.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:33:45.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:33:45.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:33:45.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:33:45.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:33:45.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:33:45.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:33:45.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:33:45.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:33:45.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:33:45.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:33:45.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:33:45.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:33:45.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:33:45.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:33:45.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:33:45.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:33:45.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:33:45.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:33:45.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:45.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:45.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:33:45.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:33:45.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:33:45.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:33:45.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:33:45.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:33:45.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:33:45.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:33:45.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:33:45.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:33:45.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:33:45.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:33:45.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:45.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:45.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:45 smithi172 ceph-mon[22747]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:33:45.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:33:45.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:33:45.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:33:45.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:33:45.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:45.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:45.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:33:45.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:33:45.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:33:45.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:33:45.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:33:45.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:33:45.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:33:45.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:33:45.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:33:45.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:33:45.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:33:45.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:33:45.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:45.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:45.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:33:45.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:33:45.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:33:45.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:33:45.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:33:45.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:33:45.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:33:45.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:33:45.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:33:45.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:33:45.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:33:45.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:33:45.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:33:45.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:33:45.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:33:45.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:33:45.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:33:45.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:33:45.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:33:45.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:33:45.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:33:45.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:33:45.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:45.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:45.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:33:45.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:33:45.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:33:45.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:33:45.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:33:45.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:33:45.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:33:45.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:33:45.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:33:45.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:33:45.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:33:45.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:33:45.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:45.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:45.972 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:45 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:33:45.972 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:45 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:33:45.972 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:45 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:33:45.972 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:45 smithi172 ceph-mon[22747]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:33:45.972 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:45 smithi172 ceph-mon[22747]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-04-22T15:33:45.973 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:45 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:45.973 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:45 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:33:45.973 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:45 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:33:46.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-mon[26606]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:33:46.123 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:33:46.123 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:33:46.123 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:33:46.123 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:33:46.123 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:46.123 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:46.123 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:33:46.123 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:33:46.123 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:33:46.124 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:33:46.124 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:33:46.124 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:33:46.124 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:33:46.124 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:33:46.124 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:33:46.124 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:33:46.124 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:33:46.124 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:33:46.124 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:46.124 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:46.124 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:33:46.124 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:33:46.124 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:33:46.124 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:33:46.125 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:33:46.125 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:33:46.125 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:33:46.125 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:33:46.125 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:33:46.125 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:33:46.125 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:33:46.125 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:33:46.125 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:33:46.125 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:33:46.125 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:33:46.125 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:33:46.125 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:33:46.126 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:33:46.126 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:33:46.126 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:33:46.126 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:33:46.126 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:33:46.126 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:46.126 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:46.126 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:33:46.126 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:33:46.126 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:33:46.126 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:33:46.126 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:33:46.126 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:33:46.127 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:33:46.127 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:33:46.127 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:33:46.127 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-mon[21899]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:33:46.127 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:33:46.127 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:33:46.127 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:33:46.127 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:33:46.127 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:46.128 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:46.128 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:33:46.128 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:33:46.128 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:33:46.128 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:33:46.128 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:33:46.128 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:33:46.128 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:33:46.128 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:33:46.128 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:33:46.128 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:33:46.128 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:33:46.128 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:33:46.129 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:46.129 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:46.129 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:33:46.129 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:33:46.129 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:33:46.129 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:33:46.129 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:33:46.129 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:33:46.129 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:33:46.129 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:33:46.129 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:33:46.129 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:33:46.129 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:33:46.129 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:33:46.130 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:33:46.130 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:33:46.130 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:33:46.130 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:33:46.130 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:33:46.130 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:33:46.130 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:33:46.130 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:33:46.130 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:33:46.130 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:33:46.130 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:46.130 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:46.130 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:33:46.130 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:33:46.130 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:33:46.131 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:33:46.131 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:33:46.131 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:33:46.131 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:33:46.131 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:33:46.131 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:33:46.131 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:33:46.131 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:33:46.131 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:33:46.131 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:46.131 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:46.131 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:33:46.131 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:33:46.131 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:33:46.131 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-mon[21899]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:33:46.132 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-mon[21899]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-04-22T15:33:46.132 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:46.132 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:33:46.132 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:33:46.132 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:33:46.132 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:33:46.132 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:33:46.132 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:46.132 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:46.132 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:33:46.132 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:33:46.132 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:33:46.133 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-mon[26606]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:33:46.133 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-mon[26606]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-04-22T15:33:46.133 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:33:46.133 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:33:46.133 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:45 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:33:46.848 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:46 smithi145 ceph-mon[26606]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:33:46.848 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:46 smithi145 ceph-mon[26606]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:33:46.848 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:46 smithi145 ceph-mon[21899]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:33:46.848 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:46 smithi145 ceph-mon[21899]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:33:46.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:46 smithi172 ceph-mon[22747]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:33:46.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:46 smithi172 ceph-mon[22747]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:33:47.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:33:47.251+0000 7f42a94bd700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:33:47.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:33:47.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:33:47.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:33:47.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:33:47.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:47.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:47.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:33:47.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:33:47.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:33:47.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:33:47.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:33:47.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:33:47.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:33:47.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:33:47.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:33:47.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:33:47.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:33:47.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:33:47.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:47.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:47.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:33:47.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:33:47.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:33:47.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:33:47.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:33:47.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:33:47.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:33:47.378 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:33:47.378 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:33:47.378 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:33:47.378 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:33:47.378 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:33:47.378 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:33:47.378 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:33:47.378 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:33:47.379 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:33:47.379 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:33:47.379 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:33:47.379 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:33:47.379 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:33:47.379 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:33:47.379 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:33:47.380 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:47.380 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:47.380 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:33:47.380 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:33:47.380 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:33:47.380 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:33:47.380 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:33:47.380 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:33:47.380 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:33:47.380 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:33:47.380 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:33:47.380 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:33:47.380 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:33:47.380 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:33:47.381 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:47.381 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:47.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:47 smithi172 ceph-mon[22747]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:33:47.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:33:47.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:33:47.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:33:47.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:33:47.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:47.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:47.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:33:47.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:33:47.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:33:47.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:33:47.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:33:47.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:33:47.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:33:47.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:33:47.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:33:47.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:33:47.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:33:47.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:33:47.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:47.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:47.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:33:47.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:33:47.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:33:47.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:33:47.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:33:47.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:33:47.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:33:47.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:33:47.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:33:47.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:33:47.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:33:47.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:33:47.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:33:47.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:33:47.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:33:47.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:33:47.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:33:47.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:33:47.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:33:47.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:33:47.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:33:47.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:33:47.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:47.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:47.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:33:47.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:33:47.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:33:47.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:33:47.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:33:47.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:33:47.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:33:47.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:33:47.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:33:47.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:33:47.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:33:47.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:33:47.973 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:47.973 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:47.973 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:47 smithi172 ceph-mon[22747]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 881 B/s rd, 0 op/s 2024-04-22T15:33:48.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-mon[21899]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:33:48.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:33:48.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:33:48.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:33:48.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:33:48.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:48.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:48.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:33:48.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:33:48.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:33:48.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:33:48.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:33:48.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:33:48.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:33:48.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:33:48.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:33:48.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:33:48.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:33:48.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:33:48.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:48.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:48.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:33:48.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:33:48.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:33:48.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:33:48.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:33:48.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:33:48.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:33:48.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:33:48.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:33:48.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:33:48.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:33:48.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:33:48.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:33:48.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:33:48.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:33:48.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:33:48.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:33:48.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:33:48.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:33:48.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:33:48.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:33:48.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:33:48.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:48.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:48.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:33:48.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:33:48.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:33:48.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:33:48.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:33:48.127 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:33:48.127 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:33:48.127 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:33:48.127 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:33:48.127 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:33:48.127 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:33:48.127 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:33:48.127 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:48.127 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:48.127 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-mon[21899]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 881 B/s rd, 0 op/s 2024-04-22T15:33:48.128 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-mon[26606]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:33:48.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:33:48.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:33:48.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:33:48.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:33:48.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:48.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:48.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:33:48.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:33:48.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:33:48.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:33:48.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:33:48.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:33:48.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:33:48.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:33:48.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:33:48.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:33:48.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:33:48.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:33:48.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:48.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:48.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:33:48.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:33:48.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:33:48.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:33:48.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:33:48.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:33:48.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:33:48.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:33:48.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:33:48.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:33:48.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:33:48.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:33:48.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:33:48.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:33:48.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:33:48.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:33:48.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:33:48.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:33:48.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:33:48.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:33:48.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:33:48.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:33:48.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:48.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:48.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:33:48.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:33:48.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:33:48.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:33:48.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:33:48.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:33:48.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:33:48.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:33:48.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:33:48.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:33:48.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:33:48.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:33:48.132 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:33:48.132 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:33:48.132 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:47 smithi145 ceph-mon[26606]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 881 B/s rd, 0 op/s 2024-04-22T15:33:48.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:48 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:33:49.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:48 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:33:49.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:48 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:33:50.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:50 smithi145 ceph-mon[26606]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 655 B/s rd, 0 op/s 2024-04-22T15:33:50.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:50 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:33:50.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:50 smithi145 ceph-mon[21899]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 655 B/s rd, 0 op/s 2024-04-22T15:33:50.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:50 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:33:50.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:50 smithi172 ceph-mon[22747]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 655 B/s rd, 0 op/s 2024-04-22T15:33:50.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:50 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:33:52.783 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:52 smithi145 ceph-mon[21899]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:33:52.784 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:52 smithi145 ceph-mon[26606]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:33:52.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:52 smithi172 ceph-mon[22747]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:33:54.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:54 smithi145 ceph-mon[21899]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:33:54.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:54 smithi145 ceph-mon[26606]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:33:54.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:54 smithi172 ceph-mon[22747]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:33:55.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:33:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:33:55] "GET /metrics HTTP/1.1" 200 38004 "" "Prometheus/2.43.0" 2024-04-22T15:33:56.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:56 smithi145 ceph-mon[21899]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:33:56.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:56 smithi145 ceph-mon[26606]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:33:56.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:56 smithi172 ceph-mon[22747]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:33:58.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:58 smithi145 ceph-mon[21899]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:33:58.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:58 smithi145 ceph-mon[26606]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:33:58.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:58 smithi172 ceph-mon[22747]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:33:59.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:33:59 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:33:59.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:33:59 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:33:59.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:33:59 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:34:00.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:00 smithi145 ceph-mon[21899]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:00.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:00 smithi145 ceph-mon[26606]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:00.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:00 smithi172 ceph-mon[22747]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:02.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:02 smithi145 ceph-mon[21899]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:34:02.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:02 smithi145 ceph-mon[26606]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:34:02.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:02 smithi172 ceph-mon[22747]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:34:04.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:04 smithi145 ceph-mon[26606]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:04.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:04 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:34:04.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:04 smithi145 ceph-mon[21899]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:04.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:04 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:34:04.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:04 smithi172 ceph-mon[22747]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:04.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:04 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:34:05.622 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:05 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:34:05] "GET /metrics HTTP/1.1" 200 38002 "" "Prometheus/2.43.0" 2024-04-22T15:34:06.791 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:06 smithi172 ceph-mon[22747]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:34:06.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:06 smithi145 ceph-mon[21899]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:34:06.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:06 smithi145 ceph-mon[26606]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:34:08.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:08 smithi145 ceph-mon[26606]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:08.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:08 smithi145 ceph-mon[21899]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:08.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:08 smithi172 ceph-mon[22747]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:09.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:09 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:34:09.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:09 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:34:09.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:09 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:34:10.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:10 smithi145 ceph-mon[21899]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:10.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:10 smithi145 ceph-mon[26606]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:10.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:10 smithi172 ceph-mon[22747]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:12.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:12 smithi145 ceph-mon[21899]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:34:12.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:12 smithi145 ceph-mon[26606]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:34:12.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:12 smithi172 ceph-mon[22747]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:34:14.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:14 smithi145 ceph-mon[21899]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:14.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:14 smithi145 ceph-mon[26606]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:14.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:14 smithi172 ceph-mon[22747]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:15.176 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-04-22T15:34:15.485 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:34:15] "GET /metrics HTTP/1.1" 200 38002 "" "Prometheus/2.43.0" 2024-04-22T15:34:15.771 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:15 smithi145 ceph-mon[26606]: from='client.25537 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:34:15.771 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:15 smithi145 ceph-mon[21899]: from='client.25537 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:34:15.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:15 smithi172 ceph-mon[22747]: from='client.25537 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:34:16.663 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:16 smithi145 ceph-mon[26606]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:34:16.664 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:16 smithi145 ceph-mon[21899]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:34:16.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:16 smithi172 ceph-mon[22747]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:34:17.212 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:34:17.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:17 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/568442714' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:34:17.783 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:17 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/568442714' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:34:17.784 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:17 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/568442714' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:34:17.808 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph versions | jq -e '"'"'.mgr | keys'"'"' | grep $sha1' 2024-04-22T15:34:18.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:18 smithi145 ceph-mon[26606]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:18.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:18 smithi145 ceph-mon[21899]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:18.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:18 smithi172 ceph-mon[22747]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:19.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:19 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:34:19.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:19 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:34:19.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:19 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:34:19.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:19 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:34:19.902 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)" 2024-04-22T15:34:19.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:19 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:34:19.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:19 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:34:20.559 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 2'"'"'' 2024-04-22T15:34:20.861 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:20 smithi145 ceph-mon[21899]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:20.862 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:20 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2268959496' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:34:20.862 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:20 smithi145 ceph-mon[26606]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:20.862 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:20 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2268959496' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:34:20.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:20 smithi172 ceph-mon[22747]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:20.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:20 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/2268959496' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:34:22.586 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:34:22.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:22 smithi145 ceph-mon[21899]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:34:22.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:22 smithi145 ceph-mon[26606]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:34:22.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:22 smithi172 ceph-mon[22747]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:34:23.148 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 2'"'"'' 2024-04-22T15:34:23.702 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:23 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/504394415' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:34:23.702 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:23 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/504394415' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:34:23.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:23 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/504394415' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:34:24.732 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:24 smithi145 ceph-mon[26606]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:24.733 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:24 smithi145 ceph-mon[21899]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:24.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:24 smithi172 ceph-mon[22747]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:25.622 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:34:25] "GET /metrics HTTP/1.1" 200 38001 "" "Prometheus/2.43.0" 2024-04-22T15:34:26.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:26 smithi145 ceph-mon[21899]: from='client.25561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:34:26.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:26 smithi145 ceph-mon[21899]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:34:26.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:26 smithi145 ceph-mon[26606]: from='client.25561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:34:26.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:26 smithi145 ceph-mon[26606]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:34:26.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:26 smithi172 ceph-mon[22747]: from='client.25561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:34:26.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:26 smithi172 ceph-mon[22747]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:34:27.928 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:34:28.567 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch upgrade status' 2024-04-22T15:34:28.868 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:28 smithi145 ceph-mon[26606]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:28.869 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:28 smithi145 ceph-mon[21899]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:28.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:28 smithi172 ceph-mon[22747]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:29.819 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:29 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:34:29.820 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:29 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:34:29.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:29 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:34:30.527 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:34:30.528 INFO:teuthology.orchestra.run.smithi145.stdout: "target_image": null, 2024-04-22T15:34:30.528 INFO:teuthology.orchestra.run.smithi145.stdout: "in_progress": false, 2024-04-22T15:34:30.528 INFO:teuthology.orchestra.run.smithi145.stdout: "which": "", 2024-04-22T15:34:30.528 INFO:teuthology.orchestra.run.smithi145.stdout: "services_complete": [], 2024-04-22T15:34:30.528 INFO:teuthology.orchestra.run.smithi145.stdout: "progress": null, 2024-04-22T15:34:30.528 INFO:teuthology.orchestra.run.smithi145.stdout: "message": "", 2024-04-22T15:34:30.528 INFO:teuthology.orchestra.run.smithi145.stdout: "is_paused": false 2024-04-22T15:34:30.528 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:34:30.824 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:30 smithi145 ceph-mon[26606]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:30.825 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:30 smithi145 ceph-mon[21899]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:30.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:30 smithi172 ceph-mon[22747]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:31.114 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph health detail' 2024-04-22T15:34:31.753 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:31 smithi145 ceph-mon[26606]: from='client.25567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:34:31.753 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:31 smithi145 ceph-mon[21899]: from='client.25567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:34:31.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:31 smithi172 ceph-mon[22747]: from='client.25567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:34:32.681 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:32 smithi145 ceph-mon[26606]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:34:32.682 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:32 smithi145 ceph-mon[21899]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:34:32.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:32 smithi172 ceph-mon[22747]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:34:33.225 INFO:teuthology.orchestra.run.smithi145.stdout:HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:34:33.225 INFO:teuthology.orchestra.run.smithi145.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:34:33.225 INFO:teuthology.orchestra.run.smithi145.stdout: daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state 2024-04-22T15:34:33.841 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:33 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/337292130' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:34:33.841 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:33 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/337292130' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:34:33.864 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '"'"'{print $2}'"'"')' 2024-04-22T15:34:33.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:33 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/337292130' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:34:34.865 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:34 smithi145 ceph-mon[26606]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:34.865 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:34 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:34:34.865 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:34 smithi145 ceph-mon[21899]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:34.865 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:34 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:34:34.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:34 smithi172 ceph-mon[22747]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:34.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:34 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:34:35.402 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:34:35] "GET /metrics HTTP/1.1" 200 38000 "" "Prometheus/2.43.0" 2024-04-22T15:34:36.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:36 smithi145 ceph-mon[26606]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:34:36.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:36 smithi145 ceph-mon[21899]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:34:36.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:36 smithi172 ceph-mon[22747]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:34:37.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:37 smithi145 ceph-mon[26606]: from='client.25579 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:34:37.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:37 smithi145 ceph-mon[26606]: from='client.25585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "daemon_types": "mon", "hosts": "smithi172", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:34:37.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:37 smithi145 ceph-mon[21899]: from='client.25579 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:34:37.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:37 smithi145 ceph-mon[21899]: from='client.25585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "daemon_types": "mon", "hosts": "smithi172", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:34:37.921 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:37 smithi172 ceph-mon[22747]: from='client.25579 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:34:37.921 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:37 smithi172 ceph-mon[22747]: from='client.25585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "daemon_types": "mon", "hosts": "smithi172", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:34:38.294 INFO:teuthology.orchestra.run.smithi145.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:34:38.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:38 smithi145 ceph-mon[26606]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:38.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:38 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:38.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:38 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:34:38.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:38 smithi145 ceph-mon[21899]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:38.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:38 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:38.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:38 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:34:38.915 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-22T15:34:38.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:38 smithi172 ceph-mon[22747]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:38.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:38 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:38.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:38 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:34:39.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:39 smithi145 ceph-mon[21899]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:34:39.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:39 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:34:39.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:39 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:34:39.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:39 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:34:39.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:39 smithi145 ceph-mon[21899]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 883 B/s rd, 0 op/s 2024-04-22T15:34:39.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:39 smithi145 ceph-mon[21899]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:34:39.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:39 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:39.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:39 smithi145 ceph-mon[21899]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:34:39.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:39 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:34:39.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:39 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:34:39.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:39 smithi145 ceph-mon[21899]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:34:39.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:39 smithi145 ceph-mon[26606]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:34:39.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:39 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:34:39.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:39 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:34:39.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:39 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:34:39.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:39 smithi145 ceph-mon[26606]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 883 B/s rd, 0 op/s 2024-04-22T15:34:39.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:39 smithi145 ceph-mon[26606]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:34:39.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:39 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:39.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:39 smithi145 ceph-mon[26606]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:34:39.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:39 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:34:39.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:39 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:34:39.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:39 smithi145 ceph-mon[26606]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:34:39.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:39 smithi172 ceph-mon[22747]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:34:39.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:39 smithi172 ceph-mon[22747]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:34:39.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:39 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:34:39.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:39 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:34:39.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:39 smithi172 ceph-mon[22747]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 883 B/s rd, 0 op/s 2024-04-22T15:34:39.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:39 smithi172 ceph-mon[22747]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:34:39.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:39 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:39.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:39 smithi172 ceph-mon[22747]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:34:39.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:39 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:34:39.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:39 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:34:39.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:39 smithi172 ceph-mon[22747]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:34:41.128 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:34:41.798 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:34:41.798 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (93s) 88s ago 23m 16.0M - 0.25.0 c8568f914cd2 1c58ccf37ffd 2024-04-22T15:34:41.798 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (69s) 64s ago 23m 51.3M - 9.4.12 2bacad6d85d8 2f825d15ab10 2024-04-22T15:34:41.799 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 unknown 88s ago 22m 59.6M - 3.5 e1d6a67b021e 9704631ffa0e 2024-04-22T15:34:41.799 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443,9283,8765 running (2m) 64s ago 28m 452M - 19.0.0-3223-g5883d494 3d983c26e278 e7500ef71604 2024-04-22T15:34:41.799 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (12m) 88s ago 30m 505M - 19.0.0-3223-g5883d494 3d983c26e278 b2de97566cf4 2024-04-22T15:34:41.799 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (30m) 88s ago 30m 79.5M 2048M 17.2.0 e1d6a67b021e 28106a7bf16e 2024-04-22T15:34:41.799 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (28m) 64s ago 28m 59.7M 2048M 17.2.0 e1d6a67b021e a2fc3297953a 2024-04-22T15:34:41.799 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (28m) 88s ago 28m 56.7M 2048M 17.2.0 e1d6a67b021e b71d9b9f8509 2024-04-22T15:34:41.799 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (2m) 88s ago 24m 8082k - 1.5.0 0da6a335fe13 8c75c26fec05 2024-04-22T15:34:41.799 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (2m) 64s ago 24m 16.0M - 1.5.0 0da6a335fe13 8fd2949e2eaf 2024-04-22T15:34:41.799 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (28m) 88s ago 28m 56.7M 2992M 17.2.0 e1d6a67b021e ea9cdc639916 2024-04-22T15:34:41.799 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (27m) 88s ago 27m 55.2M 2992M 17.2.0 e1d6a67b021e 3b9b37a8781f 2024-04-22T15:34:41.799 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (27m) 88s ago 27m 50.8M 2992M 17.2.0 e1d6a67b021e d9502dded8f8 2024-04-22T15:34:41.799 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (26m) 88s ago 26m 53.0M 2992M 17.2.0 e1d6a67b021e 8c72f30e54fc 2024-04-22T15:34:41.799 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (26m) 64s ago 26m 57.7M 3248M 17.2.0 e1d6a67b021e 80001f97ed35 2024-04-22T15:34:41.799 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (26m) 64s ago 26m 54.2M 3248M 17.2.0 e1d6a67b021e ec3ab1335b5c 2024-04-22T15:34:41.800 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (25m) 64s ago 25m 54.6M 3248M 17.2.0 e1d6a67b021e 45d475edf1ca 2024-04-22T15:34:41.800 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (25m) 64s ago 25m 52.8M 3248M 17.2.0 e1d6a67b021e cc9e1d9e7916 2024-04-22T15:34:41.800 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (117s) 64s ago 24m 55.8M - 2.43.0 a07b618ecd1d 51cf4df6e8d4 2024-04-22T15:34:41.800 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (23m) 88s ago 23m 78.0M - 17.2.0 e1d6a67b021e c90a636c947e 2024-04-22T15:34:41.800 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (23m) 64s ago 23m 75.7M - 17.2.0 e1d6a67b021e 20ac46e9bfda 2024-04-22T15:34:42.116 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:34:41.829+0000 7f42a94bd700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:34:42.116 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:34:42.116 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:34:42.117 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:34:42.117 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:34:42.117 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:34:42.117 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:34:42.117 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:34:42.117 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:34:42.117 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:34:42.117 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:34:42.117 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:34:42.117 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:34:42.117 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:34:42.117 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:34:42.117 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:34:42.117 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:34:42.117 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:34:42.118 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:34:42.118 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:34:42.118 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:34:42.118 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:34:42.118 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:34:42.118 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:34:42.119 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:34:42.119 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:34:42.119 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:34:42.119 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:34:42.119 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:34:42.119 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:34:42.119 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:34:42.119 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:34:42.119 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:34:42.119 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:34:42.119 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:34:42.119 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:34:42.119 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:34:42.119 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:34:42.119 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:34:42.119 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:34:42.120 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:34:42.120 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:34:42.120 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:34:42.120 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:34:42.120 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:34:42.120 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:34:42.120 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:34:42.120 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:34:42.120 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:34:42.120 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:34:42.120 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:34:42.120 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:34:42.120 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:34:42.120 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:34:42.120 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:34:42.121 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:34:42.121 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:34:42.121 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:34:42.121 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:34:42.222 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:34:42.222 INFO:teuthology.orchestra.run.smithi145.stdout: "mon": { 2024-04-22T15:34:42.222 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-22T15:34:42.222 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:34:42.222 INFO:teuthology.orchestra.run.smithi145.stdout: "mgr": { 2024-04-22T15:34:42.223 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:34:42.223 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:34:42.223 INFO:teuthology.orchestra.run.smithi145.stdout: "osd": { 2024-04-22T15:34:42.223 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T15:34:42.223 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:34:42.223 INFO:teuthology.orchestra.run.smithi145.stdout: "mds": {}, 2024-04-22T15:34:42.223 INFO:teuthology.orchestra.run.smithi145.stdout: "rgw": { 2024-04-22T15:34:42.223 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:34:42.223 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:34:42.223 INFO:teuthology.orchestra.run.smithi145.stdout: "overall": { 2024-04-22T15:34:42.224 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-22T15:34:42.225 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:34:42.225 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:34:42.225 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:34:42.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:42 smithi145 ceph-mon[26606]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:34:42.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:42 smithi145 ceph-mon[26606]: from='client.15669 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:34:42.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:42 smithi145 ceph-mon[26606]: from='client.15675 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:34:42.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:42 smithi145 ceph-mon[21899]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:34:42.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:42 smithi145 ceph-mon[21899]: from='client.15669 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:34:42.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:42 smithi145 ceph-mon[21899]: from='client.15675 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:34:42.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:42 smithi172 ceph-mon[22747]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:34:42.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:42 smithi172 ceph-mon[22747]: from='client.15669 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:34:42.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:42 smithi172 ceph-mon[22747]: from='client.15675 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:34:42.612 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:34:42.612 INFO:teuthology.orchestra.run.smithi145.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", 2024-04-22T15:34:42.612 INFO:teuthology.orchestra.run.smithi145.stdout: "in_progress": true, 2024-04-22T15:34:42.612 INFO:teuthology.orchestra.run.smithi145.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi172", 2024-04-22T15:34:42.612 INFO:teuthology.orchestra.run.smithi145.stdout: "services_complete": [], 2024-04-22T15:34:42.612 INFO:teuthology.orchestra.run.smithi145.stdout: "progress": "", 2024-04-22T15:34:42.613 INFO:teuthology.orchestra.run.smithi145.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa image", 2024-04-22T15:34:42.613 INFO:teuthology.orchestra.run.smithi145.stdout: "is_paused": false 2024-04-22T15:34:42.613 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:34:43.292 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:43 smithi145 ceph-mon[21899]: from='client.25603 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:34:43.293 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:43 smithi145 ceph-mon[21899]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:34:43.293 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:34:43.293 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:34:43.293 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:34:43.293 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:34:43.293 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:34:43.293 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:34:43.293 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:34:43.293 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:34:43.293 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:34:43.293 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:34:43.293 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:34:43.294 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:34:43.294 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:34:43.294 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:34:43.294 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:34:43.294 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:34:43.294 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:34:43.294 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:34:43.294 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:34:43.294 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:34:43.294 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:34:43.294 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:34:43.294 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:34:43.294 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:34:43.294 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:34:43.294 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:34:43.295 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:34:43.295 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:34:43.295 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:34:43.295 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:34:43.295 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:34:43.295 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:34:43.295 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:34:43.295 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:34:43.295 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:34:43.295 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:34:43.295 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:34:43.295 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:34:43.295 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:34:43.295 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:34:43.295 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:34:43.296 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:34:43.296 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:34:43.296 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:34:43.296 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:34:43.296 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:34:43.296 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:34:43.296 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:34:43.296 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:34:43.296 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:34:43.296 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:34:43.296 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:34:43.296 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:34:43.296 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:43 smithi145 ceph-mon[26606]: from='client.25603 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:34:43.297 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:43 smithi145 ceph-mon[26606]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:34:43.297 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:34:43.297 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:34:43.297 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:34:43.297 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:34:43.297 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:34:43.297 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:34:43.297 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:34:43.297 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:34:43.297 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:34:43.297 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:34:43.297 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:34:43.297 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:34:43.297 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:34:43.297 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:34:43.297 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:34:43.298 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:34:43.298 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:34:43.298 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:34:43.298 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:34:43.298 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:34:43.298 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:34:43.298 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:34:43.298 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:34:43.298 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:34:43.298 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:34:43.298 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:34:43.298 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:34:43.298 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:34:43.298 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:34:43.299 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:34:43.299 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:34:43.299 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:34:43.299 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:34:43.299 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:34:43.299 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:34:43.299 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:34:43.299 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:34:43.299 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:34:43.299 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:34:43.299 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:34:43.299 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:34:43.299 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:34:43.299 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:34:43.299 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:34:43.300 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:34:43.300 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:34:43.300 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:34:43.300 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:34:43.300 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:34:43.300 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:34:43.300 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:34:43.300 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:34:43.300 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:34:43.300 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:34:43.301 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:34:43.301 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:34:43.301 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:34:43.301 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:34:43.301 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:43 smithi145 ceph-mon[21899]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:34:43.301 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:43 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/1153904428' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:34:43.301 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:34:43.301 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:34:43.301 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:34:43.301 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:34:43.301 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:34:43.301 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:43 smithi145 ceph-mon[26606]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:34:43.302 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:43 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/1153904428' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:34:43.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:43 smithi172 ceph-mon[22747]: from='client.25603 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:34:43.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:43 smithi172 ceph-mon[22747]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:34:43.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:34:43.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:34:43.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:34:43.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:34:43.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:34:43.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:34:43.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:34:43.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:34:43.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:34:43.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:34:43.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:34:43.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:34:43.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:34:43.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:34:43.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:34:43.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:34:43.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:34:43.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:34:43.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:34:43.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:34:43.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:34:43.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:34:43.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:34:43.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:34:43.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:34:43.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:34:43.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:34:43.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:34:43.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:34:43.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:34:43.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:34:43.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:34:43.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:34:43.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:34:43.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:34:43.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:34:43.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:34:43.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:34:43.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:34:43.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:34:43.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:34:43.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:34:43.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:34:43.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:34:43.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:34:43.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:34:43.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:34:43.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:34:43.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:34:43.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:34:43.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:34:43.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:34:43.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:34:43.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:34:43.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:34:43.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:34:43.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:34:43.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:34:43.472 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:43 smithi172 ceph-mon[22747]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:34:43.472 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:43 smithi172 ceph-mon[22747]: from='client.? 172.21.15.145:0/1153904428' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:34:44.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:44 smithi145 ceph-mon[21899]: from='client.15693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:34:44.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:44 smithi145 ceph-mon[21899]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:34:44.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:44 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:44.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:44 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:34:44.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:44 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:34:44.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:44 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:44.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:44 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-22T15:34:44.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:44 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-22T15:34:44.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:44 smithi145 ceph-mon[26606]: from='client.15693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:34:44.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:44 smithi145 ceph-mon[26606]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:34:44.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:44 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:44.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:44 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:34:44.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:44 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:34:44.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:44 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:44.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:44 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-22T15:34:44.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:44 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-22T15:34:44.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:44 smithi172 ceph-mon[22747]: from='client.15693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:34:44.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:44 smithi172 ceph-mon[22747]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:34:44.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:44 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:44.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:44 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:34:44.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:44 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:34:44.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:44 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:44.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:44 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-22T15:34:44.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:44 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-22T15:34:45.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:45 smithi172 ceph-mon[22747]: Upgrade: Target is version 19.0.0-3223-g5883d494 (squid) 2024-04-22T15:34:45.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:45 smithi172 ceph-mon[22747]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, digests ['quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb'] 2024-04-22T15:34:45.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:45 smithi172 ceph-mon[22747]: Upgrade: Setting container_image for all mgr 2024-04-22T15:34:45.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:45 smithi172 ceph-mon[22747]: Upgrade: It appears safe to stop mon.b 2024-04-22T15:34:45.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:45 smithi145 ceph-mon[21899]: Upgrade: Target is version 19.0.0-3223-g5883d494 (squid) 2024-04-22T15:34:45.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:45 smithi145 ceph-mon[21899]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, digests ['quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb'] 2024-04-22T15:34:45.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:45 smithi145 ceph-mon[21899]: Upgrade: Setting container_image for all mgr 2024-04-22T15:34:45.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:45 smithi145 ceph-mon[21899]: Upgrade: It appears safe to stop mon.b 2024-04-22T15:34:45.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:34:45] "GET /metrics HTTP/1.1" 200 38000 "" "Prometheus/2.43.0" 2024-04-22T15:34:45.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:45 smithi145 ceph-mon[26606]: Upgrade: Target is version 19.0.0-3223-g5883d494 (squid) 2024-04-22T15:34:45.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:45 smithi145 ceph-mon[26606]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, digests ['quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb'] 2024-04-22T15:34:45.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:45 smithi145 ceph-mon[26606]: Upgrade: Setting container_image for all mgr 2024-04-22T15:34:45.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:45 smithi145 ceph-mon[26606]: Upgrade: It appears safe to stop mon.b 2024-04-22T15:34:46.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:46 smithi172 ceph-mon[22747]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 533 B/s rd, 0 op/s 2024-04-22T15:34:46.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:46 smithi172 ceph-mon[22747]: Upgrade: Updating mon.b 2024-04-22T15:34:46.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:46 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:46.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:46 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:34:46.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:46 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:34:46.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:46 smithi172 ceph-mon[22747]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:34:46.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:46 smithi172 ceph-mon[22747]: Deploying daemon mon.b on smithi172 2024-04-22T15:34:46.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:46 smithi145 ceph-mon[21899]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 533 B/s rd, 0 op/s 2024-04-22T15:34:46.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:46 smithi145 ceph-mon[21899]: Upgrade: Updating mon.b 2024-04-22T15:34:46.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:46 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:46.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:46 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:34:46.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:46 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:34:46.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:46 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:34:46.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:46 smithi145 ceph-mon[21899]: Deploying daemon mon.b on smithi172 2024-04-22T15:34:46.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:46 smithi145 ceph-mon[26606]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 533 B/s rd, 0 op/s 2024-04-22T15:34:46.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:46 smithi145 ceph-mon[26606]: Upgrade: Updating mon.b 2024-04-22T15:34:46.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:46 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:46.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:46 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:34:46.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:46 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:34:46.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:46 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:34:46.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:46 smithi145 ceph-mon[26606]: Deploying daemon mon.b on smithi172 2024-04-22T15:34:47.491 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:47 smithi172 systemd[1]: Stopping Ceph mon.b for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:34:47.790 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-b[22728]: 2024-04-22T15:34:47.487+0000 7f60457ca700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:34:47.790 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-b[22728]: 2024-04-22T15:34:47.487+0000 7f60457ca700 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2024-04-22T15:34:47.790 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:47 smithi172 podman[65142]: 2024-04-22 15:34:47.518870403 +0000 UTC m=+0.144898354 container died a2fc3297953a9de93e007b77fdd15db0bfd523c2a04fe6c7a0cc6f352f167da4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-b, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , ceph=True, distribution-scope=public, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-22T15:34:47.790 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:47 smithi172 podman[65142]: 2024-04-22 15:34:47.787538829 +0000 UTC m=+0.413566778 container cleanup a2fc3297953a9de93e007b77fdd15db0bfd523c2a04fe6c7a0cc6f352f167da4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-b, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-22T15:34:47.790 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:47 smithi172 bash[65142]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-b 2024-04-22T15:34:48.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:48 smithi172 podman[65156]: 2024-04-22 15:34:48.164132655 +0000 UTC m=+0.647496199 container remove a2fc3297953a9de93e007b77fdd15db0bfd523c2a04fe6c7a0cc6f352f167da4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-b, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, ceph=True, release=754, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, architecture=x86_64, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-22T15:34:48.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:48 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mon.b.service: Deactivated successfully. 2024-04-22T15:34:48.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:48 smithi172 systemd[1]: Stopped Ceph mon.b for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:34:48.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:48 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mon.b.service: Consumed 18.290s CPU time. 2024-04-22T15:34:48.876 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:48 smithi172 systemd[1]: Starting Ceph mon.b for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:34:48.877 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:48 smithi172 podman[65287]: 2024-04-22 15:34:48.664426828 +0000 UTC m=+0.018515363 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:34:48.877 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:48 smithi172 podman[65287]: 2024-04-22 15:34:48.773417999 +0000 UTC m=+0.127506520 container create 0a360b08a6d618e8724ec24b720135e49374ef5b5ec35dab2de8a8f4c2e8c15e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-b, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-22T15:34:49.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:48 smithi172 podman[65287]: 2024-04-22 15:34:48.984232313 +0000 UTC m=+0.338320834 container init 0a360b08a6d618e8724ec24b720135e49374ef5b5ec35dab2de8a8f4c2e8c15e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-b, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-04-22T15:34:49.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:48 smithi172 podman[65287]: 2024-04-22 15:34:48.987399986 +0000 UTC m=+0.341488511 container start 0a360b08a6d618e8724ec24b720135e49374ef5b5ec35dab2de8a8f4c2e8c15e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-b, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-04-22T15:34:49.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: set uid:gid to 167:167 (ceph:ceph) 2024-04-22T15:34:49.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev), process ceph-mon, pid 2 2024-04-22T15:34:49.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: pidfile_write: ignore empty --pid-file 2024-04-22T15:34:49.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: load: jerasure load: lrc 2024-04-22T15:34:49.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: RocksDB version: 7.9.2 2024-04-22T15:34:49.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Git sha 0 2024-04-22T15:34:49.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Compile date 2024-04-22 12:55:29 2024-04-22T15:34:49.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: DB SUMMARY 2024-04-22T15:34:49.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: DB Session ID: 23K8J6XMMWV5GHAG66KE 2024-04-22T15:34:49.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: CURRENT file: CURRENT 2024-04-22T15:34:49.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: IDENTITY file: IDENTITY 2024-04-22T15:34:49.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: MANIFEST file: MANIFEST-000009 size: 2814 Bytes 2024-04-22T15:34:49.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000054.sst 2024-04-22T15:34:49.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000052.log size: 3027601 ; 2024-04-22T15:34:49.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.error_if_exists: 0 2024-04-22T15:34:49.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.create_if_missing: 0 2024-04-22T15:34:49.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.paranoid_checks: 1 2024-04-22T15:34:49.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-22T15:34:49.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-22T15:34:49.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-22T15:34:49.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.env: 0x564409faccc0 2024-04-22T15:34:49.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.fs: PosixFileSystem 2024-04-22T15:34:49.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.info_log: 0x56440c431be0 2024-04-22T15:34:49.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.max_file_opening_threads: 16 2024-04-22T15:34:49.221 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.statistics: (nil) 2024-04-22T15:34:49.221 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.use_fsync: 0 2024-04-22T15:34:49.221 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.max_log_file_size: 0 2024-04-22T15:34:49.221 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-22T15:34:49.221 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-22T15:34:49.221 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.keep_log_file_num: 1000 2024-04-22T15:34:49.221 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.recycle_log_file_num: 0 2024-04-22T15:34:49.221 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.allow_fallocate: 1 2024-04-22T15:34:49.221 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.allow_mmap_reads: 0 2024-04-22T15:34:49.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.allow_mmap_writes: 0 2024-04-22T15:34:49.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.use_direct_reads: 0 2024-04-22T15:34:49.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-22T15:34:49.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.create_missing_column_families: 0 2024-04-22T15:34:49.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.db_log_dir: 2024-04-22T15:34:49.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.wal_dir: 2024-04-22T15:34:49.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-22T15:34:49.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-22T15:34:49.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-22T15:34:49.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-22T15:34:49.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-22T15:34:49.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-22T15:34:49.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.advise_random_on_open: 1 2024-04-22T15:34:49.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.db_write_buffer_size: 0 2024-04-22T15:34:49.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.write_buffer_manager: 0x56440c3c5220 2024-04-22T15:34:49.223 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-22T15:34:49.223 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-22T15:34:49.223 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-22T15:34:49.223 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.rate_limiter: (nil) 2024-04-22T15:34:49.223 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-22T15:34:49.223 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.wal_recovery_mode: 2 2024-04-22T15:34:49.223 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.enable_thread_tracking: 0 2024-04-22T15:34:49.223 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.enable_pipelined_write: 0 2024-04-22T15:34:49.223 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.unordered_write: 0 2024-04-22T15:34:49.223 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-22T15:34:49.223 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-22T15:34:49.223 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-22T15:34:49.223 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-22T15:34:49.223 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.row_cache: None 2024-04-22T15:34:49.223 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.wal_filter: None 2024-04-22T15:34:49.223 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-22T15:34:49.224 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.allow_ingest_behind: 0 2024-04-22T15:34:49.224 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.two_write_queues: 0 2024-04-22T15:34:49.224 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.manual_wal_flush: 0 2024-04-22T15:34:49.224 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.wal_compression: 0 2024-04-22T15:34:49.224 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.atomic_flush: 0 2024-04-22T15:34:49.224 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-22T15:34:49.224 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-22T15:34:49.224 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-22T15:34:49.224 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.log_readahead_size: 0 2024-04-22T15:34:49.224 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-22T15:34:49.224 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.best_efforts_recovery: 0 2024-04-22T15:34:49.224 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-22T15:34:49.224 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-22T15:34:49.224 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.allow_data_in_errors: 0 2024-04-22T15:34:49.224 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.db_host_id: __hostname__ 2024-04-22T15:34:49.225 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-22T15:34:49.225 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.max_background_jobs: 2 2024-04-22T15:34:49.225 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.max_background_compactions: -1 2024-04-22T15:34:49.225 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.max_subcompactions: 1 2024-04-22T15:34:49.225 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-22T15:34:49.225 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-22T15:34:49.225 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-22T15:34:49.225 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.max_total_wal_size: 0 2024-04-22T15:34:49.225 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-22T15:34:49.225 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-22T15:34:49.225 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-22T15:34:49.225 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-22T15:34:49.225 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.max_open_files: -1 2024-04-22T15:34:49.225 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.bytes_per_sync: 0 2024-04-22T15:34:49.225 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-22T15:34:49.225 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-22T15:34:49.226 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.compaction_readahead_size: 0 2024-04-22T15:34:49.226 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.max_background_flushes: -1 2024-04-22T15:34:49.226 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Compression algorithms supported: 2024-04-22T15:34:49.226 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-22T15:34:49.226 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: kZSTD supported: 0 2024-04-22T15:34:49.226 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: kXpressCompression supported: 0 2024-04-22T15:34:49.226 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: kLZ4HCCompression supported: 1 2024-04-22T15:34:49.226 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: kZlibCompression supported: 1 2024-04-22T15:34:49.226 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: kSnappyCompression supported: 1 2024-04-22T15:34:49.226 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: kLZ4Compression supported: 1 2024-04-22T15:34:49.226 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: kBZip2Compression supported: 0 2024-04-22T15:34:49.226 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-22T15:34:49.226 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-22T15:34:49.226 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-04-22T15:34:49.227 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-22T15:34:49.227 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-22T15:34:49.227 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.merge_operator: 2024-04-22T15:34:49.227 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.compaction_filter: None 2024-04-22T15:34:49.227 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.compaction_filter_factory: None 2024-04-22T15:34:49.227 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.sst_partitioner_factory: None 2024-04-22T15:34:49.227 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-22T15:34:49.227 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-22T15:34:49.227 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56440c431dc0) 2024-04-22T15:34:49.227 INFO:journalctl@ceph.mon.b.smithi172.stdout: cache_index_and_filter_blocks: 1 2024-04-22T15:34:49.227 INFO:journalctl@ceph.mon.b.smithi172.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-22T15:34:49.227 INFO:journalctl@ceph.mon.b.smithi172.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-22T15:34:49.227 INFO:journalctl@ceph.mon.b.smithi172.stdout: pin_top_level_index_and_filter: 1 2024-04-22T15:34:49.227 INFO:journalctl@ceph.mon.b.smithi172.stdout: index_type: 0 2024-04-22T15:34:49.227 INFO:journalctl@ceph.mon.b.smithi172.stdout: data_block_index_type: 0 2024-04-22T15:34:49.228 INFO:journalctl@ceph.mon.b.smithi172.stdout: index_shortening: 1 2024-04-22T15:34:49.228 INFO:journalctl@ceph.mon.b.smithi172.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-22T15:34:49.228 INFO:journalctl@ceph.mon.b.smithi172.stdout: checksum: 4 2024-04-22T15:34:49.228 INFO:journalctl@ceph.mon.b.smithi172.stdout: no_block_cache: 0 2024-04-22T15:34:49.228 INFO:journalctl@ceph.mon.b.smithi172.stdout: block_cache: 0x56440c3f71f0 2024-04-22T15:34:49.228 INFO:journalctl@ceph.mon.b.smithi172.stdout: block_cache_name: BinnedLRUCache 2024-04-22T15:34:49.228 INFO:journalctl@ceph.mon.b.smithi172.stdout: block_cache_options: 2024-04-22T15:34:49.228 INFO:journalctl@ceph.mon.b.smithi172.stdout: capacity : 536870912 2024-04-22T15:34:49.228 INFO:journalctl@ceph.mon.b.smithi172.stdout: num_shard_bits : 4 2024-04-22T15:34:49.228 INFO:journalctl@ceph.mon.b.smithi172.stdout: strict_capacity_limit : 0 2024-04-22T15:34:49.228 INFO:journalctl@ceph.mon.b.smithi172.stdout: high_pri_pool_ratio: 0.000 2024-04-22T15:34:49.228 INFO:journalctl@ceph.mon.b.smithi172.stdout: block_cache_compressed: (nil) 2024-04-22T15:34:49.228 INFO:journalctl@ceph.mon.b.smithi172.stdout: persistent_cache: (nil) 2024-04-22T15:34:49.228 INFO:journalctl@ceph.mon.b.smithi172.stdout: block_size: 4096 2024-04-22T15:34:49.228 INFO:journalctl@ceph.mon.b.smithi172.stdout: block_size_deviation: 10 2024-04-22T15:34:49.228 INFO:journalctl@ceph.mon.b.smithi172.stdout: block_restart_interval: 16 2024-04-22T15:34:49.229 INFO:journalctl@ceph.mon.b.smithi172.stdout: index_block_restart_interval: 1 2024-04-22T15:34:49.229 INFO:journalctl@ceph.mon.b.smithi172.stdout: metadata_block_size: 4096 2024-04-22T15:34:49.229 INFO:journalctl@ceph.mon.b.smithi172.stdout: partition_filters: 0 2024-04-22T15:34:49.229 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_delta_encoding: 1 2024-04-22T15:34:49.229 INFO:journalctl@ceph.mon.b.smithi172.stdout: filter_policy: bloomfilter 2024-04-22T15:34:49.229 INFO:journalctl@ceph.mon.b.smithi172.stdout: whole_key_filtering: 1 2024-04-22T15:34:49.229 INFO:journalctl@ceph.mon.b.smithi172.stdout: verify_compression: 0 2024-04-22T15:34:49.229 INFO:journalctl@ceph.mon.b.smithi172.stdout: read_amp_bytes_per_bit: 0 2024-04-22T15:34:49.229 INFO:journalctl@ceph.mon.b.smithi172.stdout: format_version: 5 2024-04-22T15:34:49.229 INFO:journalctl@ceph.mon.b.smithi172.stdout: enable_index_compression: 1 2024-04-22T15:34:49.229 INFO:journalctl@ceph.mon.b.smithi172.stdout: block_align: 0 2024-04-22T15:34:49.229 INFO:journalctl@ceph.mon.b.smithi172.stdout: max_auto_readahead_size: 262144 2024-04-22T15:34:49.229 INFO:journalctl@ceph.mon.b.smithi172.stdout: prepopulate_block_cache: 0 2024-04-22T15:34:49.229 INFO:journalctl@ceph.mon.b.smithi172.stdout: initial_auto_readahead_size: 8192 2024-04-22T15:34:49.229 INFO:journalctl@ceph.mon.b.smithi172.stdout: num_file_reads_for_auto_readahead: 2 2024-04-22T15:34:49.229 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.write_buffer_size: 33554432 2024-04-22T15:34:49.230 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.max_write_buffer_number: 2 2024-04-22T15:34:49.230 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.compression: NoCompression 2024-04-22T15:34:49.230 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.bottommost_compression: Disabled 2024-04-22T15:34:49.230 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.prefix_extractor: nullptr 2024-04-22T15:34:49.230 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-22T15:34:49.230 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.num_levels: 7 2024-04-22T15:34:49.230 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-22T15:34:49.230 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-22T15:34:49.230 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-22T15:34:49.230 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-22T15:34:49.230 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-22T15:34:49.230 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-22T15:34:49.230 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-22T15:34:49.230 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-22T15:34:49.230 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-22T15:34:49.231 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-22T15:34:49.231 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-22T15:34:49.231 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-22T15:34:49.231 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-22T15:34:49.231 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.compression_opts.level: 32767 2024-04-22T15:34:49.231 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.compression_opts.strategy: 0 2024-04-22T15:34:49.231 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-22T15:34:49.231 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-22T15:34:49.231 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-22T15:34:49.231 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-22T15:34:49.231 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.compression_opts.enabled: false 2024-04-22T15:34:49.231 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-22T15:34:49.231 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-22T15:34:49.231 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-22T15:34:49.231 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-22T15:34:49.232 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.target_file_size_base: 67108864 2024-04-22T15:34:49.232 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-22T15:34:49.232 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-22T15:34:49.232 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-22T15:34:49.232 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-22T15:34:49.232 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-22T15:34:49.232 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-22T15:34:49.233 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-22T15:34:49.233 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-22T15:34:49.233 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-22T15:34:49.233 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-22T15:34:49.233 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-22T15:34:49.233 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-22T15:34:49.233 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-22T15:34:49.233 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-22T15:34:49.233 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.arena_block_size: 1048576 2024-04-22T15:34:49.233 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-22T15:34:49.233 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-22T15:34:49.234 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.disable_auto_compactions: 0 2024-04-22T15:34:49.234 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-22T15:34:49.234 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-22T15:34:49.234 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-22T15:34:49.234 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-22T15:34:49.234 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-22T15:34:49.234 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-22T15:34:49.234 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-22T15:34:49.234 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-22T15:34:49.234 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-22T15:34:49.234 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-22T15:34:49.234 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-04-22T15:34:49.234 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.inplace_update_support: 0 2024-04-22T15:34:49.234 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-22T15:34:49.235 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-22T15:34:49.235 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-22T15:34:49.235 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-22T15:34:49.235 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.bloom_locality: 0 2024-04-22T15:34:49.235 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.max_successive_merges: 0 2024-04-22T15:34:49.235 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-22T15:34:49.235 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.paranoid_file_checks: 0 2024-04-22T15:34:49.235 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.force_consistency_checks: 1 2024-04-22T15:34:49.235 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.report_bg_io_stats: 0 2024-04-22T15:34:49.235 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.ttl: 2592000 2024-04-22T15:34:49.235 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-22T15:34:49.235 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-22T15:34:49.235 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-22T15:34:49.235 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.enable_blob_files: false 2024-04-22T15:34:49.235 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.min_blob_size: 0 2024-04-22T15:34:49.235 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.blob_file_size: 268435456 2024-04-22T15:34:49.236 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-22T15:34:49.236 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-22T15:34:49.236 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-22T15:34:49.236 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-22T15:34:49.236 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-22T15:34:49.236 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.blob_file_starting_level: 0 2024-04-22T15:34:49.236 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-22T15:34:49.236 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 54.sst 2024-04-22T15:34:49.236 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-22T15:34:49.236 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 56, last_sequence is 28932, log_number is 52,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-22T15:34:49.236 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 52 2024-04-22T15:34:49.236 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 648dc050-47c2-4716-adff-c028a50c0c24 2024-04-22T15:34:49.236 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713800089016950, "job": 1, "event": "recovery_started", "wal_files": [52]} 2024-04-22T15:34:49.236 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #52 mode 2 2024-04-22T15:34:49.236 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713800089024604, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 57, "file_size": 1883252, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 28937, "largest_seqno": 30689, "table_properties": {"data_size": 1876446, "index_size": 4101, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1669, "raw_key_size": 15849, "raw_average_key_size": 24, "raw_value_size": 1862471, "raw_average_value_size": 2830, "num_data_blocks": 184, "num_entries": 658, "num_filter_entries": 658, "num_deletions": 7, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1713800089, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "648dc050-47c2-4716-adff-c028a50c0c24", "db_session_id": "23K8J6XMMWV5GHAG66KE", "orig_file_number": 57, "seqno_to_time_mapping": "N/A"}} 2024-04-22T15:34:49.237 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713800089024702, "job": 1, "event": "recovery_finished"} 2024-04-22T15:34:49.237 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: [db/version_set.cc:5047] Creating manifest 59 2024-04-22T15:34:49.237 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-22T15:34:49.237 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000052.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-22T15:34:49.237 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x56440c50c000 2024-04-22T15:34:49.237 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: DB pointer 0x56440c4f6000 2024-04-22T15:34:49.237 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-22T15:34:49.237 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-22T15:34:49.237 INFO:journalctl@ceph.mon.b.smithi172.stdout: ** DB Stats ** 2024-04-22T15:34:49.237 INFO:journalctl@ceph.mon.b.smithi172.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T15:34:49.237 INFO:journalctl@ceph.mon.b.smithi172.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-22T15:34:49.237 INFO:journalctl@ceph.mon.b.smithi172.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-22T15:34:49.237 INFO:journalctl@ceph.mon.b.smithi172.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T15:34:49.237 INFO:journalctl@ceph.mon.b.smithi172.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-22T15:34:49.237 INFO:journalctl@ceph.mon.b.smithi172.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-22T15:34:49.238 INFO:journalctl@ceph.mon.b.smithi172.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T15:34:49.238 INFO:journalctl@ceph.mon.b.smithi172.stdout: 2024-04-22T15:34:49.238 INFO:journalctl@ceph.mon.b.smithi172.stdout: ** Compaction Stats [default] ** 2024-04-22T15:34:49.238 INFO:journalctl@ceph.mon.b.smithi172.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-22T15:34:49.238 INFO:journalctl@ceph.mon.b.smithi172.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-22T15:34:49.238 INFO:journalctl@ceph.mon.b.smithi172.stdout: L0 1/0 1.80 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 557.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-22T15:34:49.238 INFO:journalctl@ceph.mon.b.smithi172.stdout: L6 1/0 9.85 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-22T15:34:49.238 INFO:journalctl@ceph.mon.b.smithi172.stdout: Sum 2/0 11.64 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 557.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-22T15:34:49.238 INFO:journalctl@ceph.mon.b.smithi172.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 557.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-22T15:34:49.238 INFO:journalctl@ceph.mon.b.smithi172.stdout: 2024-04-22T15:34:49.238 INFO:journalctl@ceph.mon.b.smithi172.stdout: ** Compaction Stats [default] ** 2024-04-22T15:34:49.238 INFO:journalctl@ceph.mon.b.smithi172.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-22T15:34:49.238 INFO:journalctl@ceph.mon.b.smithi172.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T15:34:49.238 INFO:journalctl@ceph.mon.b.smithi172.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 557.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-22T15:34:49.239 INFO:journalctl@ceph.mon.b.smithi172.stdout: 2024-04-22T15:34:49.239 INFO:journalctl@ceph.mon.b.smithi172.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-22T15:34:49.239 INFO:journalctl@ceph.mon.b.smithi172.stdout: 2024-04-22T15:34:49.239 INFO:journalctl@ceph.mon.b.smithi172.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T15:34:49.239 INFO:journalctl@ceph.mon.b.smithi172.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-04-22T15:34:49.239 INFO:journalctl@ceph.mon.b.smithi172.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-22T15:34:49.239 INFO:journalctl@ceph.mon.b.smithi172.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-22T15:34:49.239 INFO:journalctl@ceph.mon.b.smithi172.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-22T15:34:49.239 INFO:journalctl@ceph.mon.b.smithi172.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-22T15:34:49.239 INFO:journalctl@ceph.mon.b.smithi172.stdout: Cumulative compaction: 0.00 GB write, 180.03 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T15:34:49.239 INFO:journalctl@ceph.mon.b.smithi172.stdout: Interval compaction: 0.00 GB write, 180.03 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T15:34:49.239 INFO:journalctl@ceph.mon.b.smithi172.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-22T15:34:49.239 INFO:journalctl@ceph.mon.b.smithi172.stdout: Block cache BinnedLRUCache@0x56440c3f71f0#2 capacity: 512.00 MB usage: 6.36 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2024-04-22T15:34:49.239 INFO:journalctl@ceph.mon.b.smithi172.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.78 KB,0.000339746%) IndexBlock(1,4.58 KB,0.000873208%) Misc(1,0.00 KB,0%) 2024-04-22T15:34:49.239 INFO:journalctl@ceph.mon.b.smithi172.stdout: 2024-04-22T15:34:49.240 INFO:journalctl@ceph.mon.b.smithi172.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-22T15:34:49.240 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: starting mon.b rank 2 at public addrs [v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0] at bind addrs [v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:34:49.240 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: mon.b@-1(???) e3 preinit fsid 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:34:49.240 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: mon.b@-1(???).mds e1 new map 2024-04-22T15:34:49.240 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: mon.b@-1(???).mds e1 print_map 2024-04-22T15:34:49.240 INFO:journalctl@ceph.mon.b.smithi172.stdout: e1 2024-04-22T15:34:49.240 INFO:journalctl@ceph.mon.b.smithi172.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-22T15:34:49.240 INFO:journalctl@ceph.mon.b.smithi172.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-22T15:34:49.240 INFO:journalctl@ceph.mon.b.smithi172.stdout: legacy client fscid: -1 2024-04-22T15:34:49.240 INFO:journalctl@ceph.mon.b.smithi172.stdout: 2024-04-22T15:34:49.240 INFO:journalctl@ceph.mon.b.smithi172.stdout: No filesystems configured 2024-04-22T15:34:49.240 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: mon.b@-1(???).osd e107 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-22T15:34:49.240 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: mon.b@-1(???).osd e107 crush map has features 432629239337189376, adjusting msgr requires 2024-04-22T15:34:49.240 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: mon.b@-1(???).osd e107 crush map has features 432629239337189376, adjusting msgr requires 2024-04-22T15:34:49.240 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: mon.b@-1(???).osd e107 crush map has features 432629239337189376, adjusting msgr requires 2024-04-22T15:34:49.240 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 ceph-mon[65307]: mon.b@-1(???).paxosservice(auth 1..25) refresh upgraded, format 0 -> 3 2024-04-22T15:34:49.241 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 bash[65287]: 0a360b08a6d618e8724ec24b720135e49374ef5b5ec35dab2de8a8f4c2e8c15e 2024-04-22T15:34:49.241 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:49 smithi172 systemd[1]: Started Ceph mon.b for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:34:50.384 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:50 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:34:50.384 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:50 smithi172 ceph-mon[65307]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:34:50.385 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:50 smithi172 ceph-mon[65307]: mon.b calling monitor election 2024-04-22T15:34:50.385 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:50 smithi172 ceph-mon[65307]: mon.c calling monitor election 2024-04-22T15:34:50.385 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:50 smithi172 ceph-mon[65307]: mon.a calling monitor election 2024-04-22T15:34:50.385 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:50 smithi172 ceph-mon[65307]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-22T15:34:50.385 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:50 smithi172 ceph-mon[65307]: monmap e3: 3 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0],b=[v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0],c=[v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0]} 2024-04-22T15:34:50.385 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:50 smithi172 ceph-mon[65307]: fsmap 2024-04-22T15:34:50.385 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:50 smithi172 ceph-mon[65307]: osdmap e107: 8 total, 8 up, 8 in 2024-04-22T15:34:50.385 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:50 smithi172 ceph-mon[65307]: mgrmap e43: y(active, since 3m), standbys: x 2024-04-22T15:34:50.385 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:50 smithi172 ceph-mon[65307]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:34:50.385 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:50 smithi172 ceph-mon[65307]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:34:50.385 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:50 smithi172 ceph-mon[65307]: daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state 2024-04-22T15:34:50.385 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:50 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:50.385 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:50 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:50.385 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:50 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:34:50.386 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:50 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:34:50.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:34:50.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[26606]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:34:50.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[26606]: mon.b calling monitor election 2024-04-22T15:34:50.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[26606]: mon.c calling monitor election 2024-04-22T15:34:50.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[26606]: mon.a calling monitor election 2024-04-22T15:34:50.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[26606]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-22T15:34:50.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[26606]: monmap e3: 3 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0],b=[v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0],c=[v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0]} 2024-04-22T15:34:50.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[26606]: fsmap 2024-04-22T15:34:50.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[26606]: osdmap e107: 8 total, 8 up, 8 in 2024-04-22T15:34:50.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[26606]: mgrmap e43: y(active, since 3m), standbys: x 2024-04-22T15:34:50.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[26606]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:34:50.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[26606]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:34:50.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[26606]: daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state 2024-04-22T15:34:50.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:50.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:50.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:34:50.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:34:50.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:34:50.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[21899]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:34:50.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[21899]: mon.b calling monitor election 2024-04-22T15:34:50.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[21899]: mon.c calling monitor election 2024-04-22T15:34:50.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[21899]: mon.a calling monitor election 2024-04-22T15:34:50.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[21899]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-22T15:34:50.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[21899]: monmap e3: 3 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0],b=[v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0],c=[v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0]} 2024-04-22T15:34:50.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[21899]: fsmap 2024-04-22T15:34:50.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[21899]: osdmap e107: 8 total, 8 up, 8 in 2024-04-22T15:34:50.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[21899]: mgrmap e43: y(active, since 3m), standbys: x 2024-04-22T15:34:50.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[21899]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:34:50.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[21899]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:34:50.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[21899]: daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state 2024-04-22T15:34:50.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:50.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:50.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:34:50.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:50 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:34:52.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:52 smithi145 ceph-mon[21899]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:34:52.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:52 smithi145 ceph-mon[26606]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:34:52.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:52 smithi172 ceph-mon[65307]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:34:54.372 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:54 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:34:54.032+0000 7f42ac4c3700 -1 mgr.server handle_report got status from non-daemon mon.b 2024-04-22T15:34:54.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:54 smithi172 ceph-mon[65307]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:54.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:54 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:54.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:54 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:54.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:54 smithi145 ceph-mon[21899]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:54.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:54 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:54.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:54 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:54.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:54 smithi145 ceph-mon[26606]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:54.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:54 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:54.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:54 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:55.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:55 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:55.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:55 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:55.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:55 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:55.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:55 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:55.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:34:55] "GET /metrics HTTP/1.1" 200 37943 "" "Prometheus/2.43.0" 2024-04-22T15:34:55.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:55 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:55.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:55 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:56.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:56 smithi172 ceph-mon[65307]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:56.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:56 smithi145 ceph-mon[21899]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:56.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:56 smithi145 ceph-mon[26606]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:34:58.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:58 smithi145 ceph-mon[21899]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:34:58.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:58 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:58.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:58 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:58.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:58 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:34:58.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:58 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:34:58.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:58 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:58.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:58 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:34:58.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:58 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:34:58.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:58 smithi145 ceph-mon[26606]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:34:58.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:58 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:58.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:58 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:58.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:58 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:34:58.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:58 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:34:58.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:58 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:58.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:58 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:34:58.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:58 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:34:58.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:58 smithi172 ceph-mon[65307]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:34:58.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:58 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:58.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:58 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:58.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:58 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:34:58.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:58 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:34:58.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:58 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:34:58.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:58 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:34:58.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:58 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:34:59.578 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-mon[26606]: Detected new or changed devices on smithi172 2024-04-22T15:34:59.578 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-mon[26606]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 945 B/s rd, 0 op/s 2024-04-22T15:34:59.578 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-mon[26606]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:34:59.578 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-mon[26606]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:34:59.578 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-mon[26606]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:34:59.578 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:34:59.579 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-mon[21899]: Detected new or changed devices on smithi172 2024-04-22T15:34:59.579 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-mon[21899]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 945 B/s rd, 0 op/s 2024-04-22T15:34:59.579 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-mon[21899]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:34:59.579 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-mon[21899]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:34:59.579 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-mon[21899]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:34:59.579 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:34:59.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:59 smithi172 ceph-mon[65307]: Detected new or changed devices on smithi172 2024-04-22T15:34:59.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:59 smithi172 ceph-mon[65307]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 945 B/s rd, 0 op/s 2024-04-22T15:34:59.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:59 smithi172 ceph-mon[65307]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:34:59.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:59 smithi172 ceph-mon[65307]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:34:59.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:59 smithi172 ceph-mon[65307]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:34:59.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:34:59 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:34:59.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:34:59.696+0000 7f42a94bd700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:34:59.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:34:59.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:34:59.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:34:59.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:34:59.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:34:59.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:34:59.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:34:59.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:34:59.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:34:59.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:34:59.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:34:59.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:34:59.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:34:59.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:34:59.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:34:59.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:34:59.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:34:59.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:34:59.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:34:59.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:34:59.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:34:59.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:34:59.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:34:59.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:34:59.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:34:59.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:34:59.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:34:59.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:34:59.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:34:59.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:34:59.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:34:59.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:34:59.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:34:59.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:34:59.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:34:59.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:34:59.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:34:59.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:34:59.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:34:59.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:34:59.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:34:59.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:34:59.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:34:59.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:34:59.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:34:59.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:34:59.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:34:59.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:34:59.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:34:59.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:34:59.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:34:59.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:34:59.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:34:59.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:34:59.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:34:59.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:34:59.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:34:59.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:34:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:00.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:35:00.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:00.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:00.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:00.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:00.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T15:35:00.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-22T15:35:00.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:00.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:00.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:00.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T15:35:00.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T15:35:00.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T15:35:00.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T15:35:00.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T15:35:00.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T15:35:00.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T15:35:00.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T15:35:00.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:35:00.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T15:35:00.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T15:35:00.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T15:35:00.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T15:35:00.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T15:35:00.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T15:35:00.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T15:35:00.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T15:35:00.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T15:35:00.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:35:00.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T15:35:00.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T15:35:00.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T15:35:00.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T15:35:00.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T15:35:00.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T15:35:00.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T15:35:00.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:35:00.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:35:00.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:35:00.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:35:00.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:35:00.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:35:00.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T15:35:00.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T15:35:00.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:35:00.724 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:35:00.724 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:35:00.724 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:00.724 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:35:00.724 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:00 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:35:00.773 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:35:00.773 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.773 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.773 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:00.773 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.774 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:00.774 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.774 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:00.774 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.774 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:00.774 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T15:35:00.774 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-22T15:35:00.774 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.774 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:00.774 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.774 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:00.774 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.775 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:00.775 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.775 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T15:35:00.775 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.775 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T15:35:00.775 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.775 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T15:35:00.775 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.775 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T15:35:00.775 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.775 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T15:35:00.776 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:35:00.776 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.776 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.776 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:00.776 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.776 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:00.776 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.776 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:00.776 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.776 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:00.777 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T15:35:00.777 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-22T15:35:00.777 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.777 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:00.777 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.777 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:00.777 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.777 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:00.777 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.777 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T15:35:00.777 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.777 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T15:35:00.777 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.778 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T15:35:00.778 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.778 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T15:35:00.778 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.778 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T15:35:00.779 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.779 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T15:35:00.779 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T15:35:00.779 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T15:35:00.779 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:35:00.779 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T15:35:00.779 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T15:35:00.779 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T15:35:00.779 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T15:35:00.779 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T15:35:00.779 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T15:35:00.779 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T15:35:00.779 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T15:35:00.780 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T15:35:00.780 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:35:00.780 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T15:35:00.780 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T15:35:00.780 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T15:35:00.780 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T15:35:00.780 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T15:35:00.780 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T15:35:00.780 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T15:35:00.780 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:35:00.780 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:35:00.780 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:35:00.780 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:35:00.780 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:35:00.780 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:35:00.781 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T15:35:00.781 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T15:35:00.781 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:35:00.781 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:35:00.781 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:35:00.781 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:00.781 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:35:00.781 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:35:00.781 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:00.782 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-22T15:35:00.782 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T15:35:00.782 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T15:35:00.782 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:35:00.782 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T15:35:00.782 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T15:35:00.782 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T15:35:00.782 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T15:35:00.782 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T15:35:00.782 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T15:35:00.782 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T15:35:00.782 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T15:35:00.782 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T15:35:00.782 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:35:00.783 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T15:35:00.783 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T15:35:00.783 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T15:35:00.783 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T15:35:00.783 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T15:35:00.783 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T15:35:00.783 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T15:35:00.783 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:35:00.783 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:35:00.783 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:35:00.783 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:35:00.783 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:35:00.783 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:35:00.783 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T15:35:00.783 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T15:35:00.784 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:35:00.784 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:35:00.784 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:35:00.784 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:00.784 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:35:00.784 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:00 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:35:01.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:01 smithi172 ceph-mon[65307]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:35:01.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:35:01.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:35:01.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:35:01.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:35:01.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:01.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:01.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:35:01.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:35:01.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:35:01.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:35:01.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:35:01.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:35:01.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:35:01.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:35:01.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:35:01.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:35:01.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:35:01.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:35:01.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:01.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:01.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:35:01.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:35:01.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:35:01.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:35:01.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:35:01.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:35:01.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:35:01.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:35:01.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:35:01.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:35:01.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:35:01.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:35:01.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:35:01.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:35:01.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:35:01.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:35:01.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:35:01.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:35:01.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:35:01.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:35:01.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:35:01.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:35:01.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:01.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:01.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:35:01.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:35:01.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:35:01.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:35:01.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:35:01.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:35:01.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:35:01.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:35:01.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:35:01.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:35:01.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:35:01.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:35:01.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:01.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:01.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:01 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all crash 2024-04-22T15:35:01.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:01 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all mds 2024-04-22T15:35:01.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:01 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T15:35:01.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:01 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-22T15:35:01.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:01 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T15:35:01.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:01 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all nfs 2024-04-22T15:35:01.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:01 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all nvmeof 2024-04-22T15:35:01.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:01 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all node-exporter 2024-04-22T15:35:01.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:01 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all prometheus 2024-04-22T15:35:01.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:01 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all alertmanager 2024-04-22T15:35:01.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:01 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all grafana 2024-04-22T15:35:01.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:01 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all loki 2024-04-22T15:35:01.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:01 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all promtail 2024-04-22T15:35:01.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:01 smithi172 ceph-mon[65307]: Upgrade: Finalizing container_image settings 2024-04-22T15:35:01.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:01 smithi172 ceph-mon[65307]: Upgrade: Complete! 2024-04-22T15:35:01.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:01 smithi172 ceph-mon[65307]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 583 B/s rd, 0 op/s 2024-04-22T15:35:01.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:01 smithi172 ceph-mon[65307]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 755 B/s rd, 0 op/s 2024-04-22T15:35:01.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:01 smithi172 ceph-mon[65307]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:35:01.724 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:01 smithi172 ceph-mon[65307]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:35:01.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[21899]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:35:01.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:35:01.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:35:01.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:35:01.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:35:01.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:01.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:01.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:35:01.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:35:01.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:35:01.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:35:01.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:35:01.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:35:01.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:35:01.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:35:01.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:35:01.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:35:01.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:35:01.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:35:01.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:01.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:01.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:35:01.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:35:01.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:35:01.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:35:01.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:35:01.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:35:01.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:35:01.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:35:01.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:35:01.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:35:01.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:35:01.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:35:01.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:35:01.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:35:01.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:35:01.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:35:01.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:35:01.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:35:01.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:35:01.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:35:01.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:35:01.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:35:01.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:01.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:01.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:35:01.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:35:01.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:35:01.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:35:01.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:35:01.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:35:01.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:35:01.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:35:01.877 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:35:01.877 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[26606]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:35:01.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:35:01.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:35:01.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:35:01.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:35:01.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:01.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:01.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:35:01.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:35:01.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:35:01.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:35:01.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:35:01.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:35:01.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:35:01.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:35:01.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:35:01.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:35:01.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:35:01.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:35:01.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:01.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:01.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:35:01.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:35:01.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:35:01.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:35:01.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:35:01.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:35:01.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:35:01.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:35:01.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:35:01.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:35:01.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:35:01.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:35:01.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:35:01.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:35:01.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:35:01.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:35:01.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:35:01.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:35:01.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:35:01.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:35:01.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:35:01.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:35:01.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:01.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:01.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:35:01.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:35:01.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:35:01.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:35:01.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:35:01.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:35:01.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:35:01.881 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:35:01.881 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:35:01.881 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:35:01.881 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:35:01.881 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:35:01.881 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:01.881 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:01.881 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[26606]: Upgrade: Setting container_image for all crash 2024-04-22T15:35:01.881 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[26606]: Upgrade: Setting container_image for all mds 2024-04-22T15:35:01.881 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[26606]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T15:35:01.881 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[26606]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-22T15:35:01.881 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[26606]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T15:35:01.881 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[26606]: Upgrade: Setting container_image for all nfs 2024-04-22T15:35:01.881 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[26606]: Upgrade: Setting container_image for all nvmeof 2024-04-22T15:35:01.882 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[26606]: Upgrade: Setting container_image for all node-exporter 2024-04-22T15:35:01.882 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[26606]: Upgrade: Setting container_image for all prometheus 2024-04-22T15:35:01.882 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[26606]: Upgrade: Setting container_image for all alertmanager 2024-04-22T15:35:01.882 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[26606]: Upgrade: Setting container_image for all grafana 2024-04-22T15:35:01.882 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[26606]: Upgrade: Setting container_image for all loki 2024-04-22T15:35:01.882 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[26606]: Upgrade: Setting container_image for all promtail 2024-04-22T15:35:01.882 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[26606]: Upgrade: Finalizing container_image settings 2024-04-22T15:35:01.882 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[26606]: Upgrade: Complete! 2024-04-22T15:35:01.882 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[26606]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 583 B/s rd, 0 op/s 2024-04-22T15:35:01.882 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[26606]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 755 B/s rd, 0 op/s 2024-04-22T15:35:01.882 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[26606]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:35:01.882 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[26606]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:35:01.883 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:35:01.883 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:35:01.883 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:35:01.883 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:01.883 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:01.883 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[21899]: Upgrade: Setting container_image for all crash 2024-04-22T15:35:01.883 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[21899]: Upgrade: Setting container_image for all mds 2024-04-22T15:35:01.883 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[21899]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T15:35:01.883 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[21899]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-22T15:35:01.883 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[21899]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T15:35:01.883 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[21899]: Upgrade: Setting container_image for all nfs 2024-04-22T15:35:01.883 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[21899]: Upgrade: Setting container_image for all nvmeof 2024-04-22T15:35:01.883 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[21899]: Upgrade: Setting container_image for all node-exporter 2024-04-22T15:35:01.884 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[21899]: Upgrade: Setting container_image for all prometheus 2024-04-22T15:35:01.884 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[21899]: Upgrade: Setting container_image for all alertmanager 2024-04-22T15:35:01.884 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[21899]: Upgrade: Setting container_image for all grafana 2024-04-22T15:35:01.884 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[21899]: Upgrade: Setting container_image for all loki 2024-04-22T15:35:01.884 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[21899]: Upgrade: Setting container_image for all promtail 2024-04-22T15:35:01.884 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[21899]: Upgrade: Finalizing container_image settings 2024-04-22T15:35:01.884 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[21899]: Upgrade: Complete! 2024-04-22T15:35:01.884 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[21899]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 583 B/s rd, 0 op/s 2024-04-22T15:35:01.884 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[21899]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 755 B/s rd, 0 op/s 2024-04-22T15:35:01.884 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[21899]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:35:01.884 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-mon[21899]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:35:01.885 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:35:01.656+0000 7f42a94bd700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:35:01.885 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:35:01.885 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:35:01.885 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:35:01.885 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:35:01.885 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:01.885 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:01.885 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:35:01.885 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:35:01.885 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:35:01.885 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:35:01.886 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:35:01.886 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:35:01.886 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:35:01.886 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:35:01.886 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:35:01.886 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:35:01.886 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:35:01.886 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:35:01.886 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:01.886 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:01.886 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:35:01.886 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:35:01.886 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:35:01.886 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:35:01.887 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:35:01.887 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:35:01.887 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:35:01.887 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:35:01.887 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:35:01.887 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:35:01.887 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:35:01.887 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:35:01.887 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:35:01.887 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:35:01.887 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:35:01.887 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:35:01.887 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:35:01.887 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:35:01.888 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:35:01.888 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:35:01.888 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:35:01.888 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:35:01.888 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:01.888 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:01.888 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:35:01.888 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:35:01.888 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:35:01.888 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:35:01.888 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:35:01.888 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:35:01.888 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:35:01.888 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:35:01.889 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:35:01.889 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:35:01.889 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:35:01.889 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:35:01.889 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:01.889 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:02.617 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:02 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:35:02.617 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:02 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:35:02.617 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:02 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:35:02.617 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:02 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:02.617 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:02 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:35:02.618 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:02 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:35:02.618 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:02 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:35:02.618 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:02 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:35:02.618 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:02 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:35:02.618 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:02 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:02.618 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:02 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:35:02.618 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:02 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:35:02.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:02 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:35:02.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:02 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:35:02.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:02 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:35:02.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:02 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:02.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:02 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:35:02.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:02 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:35:03.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-mon[21899]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:35:03.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:35:03.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:35:03.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:35:03.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:35:03.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:03.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:03.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:35:03.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:35:03.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:35:03.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:35:03.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:35:03.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:35:03.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:35:03.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:35:03.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:35:03.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:35:03.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:35:03.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:35:03.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:03.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:03.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:35:03.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:35:03.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:35:03.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:35:03.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:35:03.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:35:03.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:35:03.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:35:03.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:35:03.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:35:03.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:35:03.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:35:03.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:35:03.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:35:03.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:35:03.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:35:03.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:35:03.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:35:03.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:35:03.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:35:03.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:35:03.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:35:03.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:03.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:03.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:35:03.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:35:03.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:35:03.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:35:03.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:35:03.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:35:03.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:35:03.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:35:03.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:35:03.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-mon[26606]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:35:03.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:35:03.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:35:03.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:35:03.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:35:03.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:03.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:03.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:35:03.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:35:03.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:35:03.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:35:03.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:35:03.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:35:03.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:35:03.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:35:03.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:35:03.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:35:03.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:35:03.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:35:03.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:03.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:03.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:35:03.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:35:03.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:35:03.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:35:03.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:35:03.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:35:03.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:35:03.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:35:03.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:35:03.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:35:03.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:35:03.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:35:03.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:35:03.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:35:03.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:35:03.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:35:03.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:35:03.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:35:03.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:35:03.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:35:03.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:35:03.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:35:03.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:03.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:03.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:35:03.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:35:03.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:35:03.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:35:03.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:35:03.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:35:03.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:35:03.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:35:03.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:35:03.633 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:35:03.633 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:35:03.633 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:35:03.633 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:03.633 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:03.634 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-mon[21899]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-04-22T15:35:03.634 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-mon[21899]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:35:03.634 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-mon[21899]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:35:03.634 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-mon[21899]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:35:03.634 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:35:03.634 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:35:03.634 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:35:03.634 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:03.634 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:03.634 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-mon[26606]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-04-22T15:35:03.634 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-mon[26606]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:35:03.635 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-mon[26606]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:35:03.635 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-mon[26606]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:35:03.635 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:35:03.438+0000 7f42a94bd700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:35:03.635 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:35:03.635 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:35:03.635 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:35:03.635 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:35:03.636 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:03.636 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:03.636 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:35:03.636 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:35:03.636 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:35:03.636 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:35:03.636 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:35:03.636 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:35:03.636 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:35:03.636 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:35:03.636 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:35:03.636 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:35:03.636 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:35:03.636 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:35:03.637 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:03.637 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:03.637 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:35:03.637 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:35:03.637 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:35:03.637 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:35:03.637 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:35:03.637 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:35:03.637 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:35:03.637 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:35:03.637 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:35:03.637 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:35:03.637 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:35:03.637 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:35:03.637 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:35:03.638 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:35:03.638 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:35:03.638 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:35:03.638 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:35:03.638 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:35:03.638 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:35:03.638 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:35:03.638 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:35:03.638 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:35:03.638 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:03.638 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:03.638 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:35:03.638 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:35:03.638 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:35:03.639 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:35:03.639 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:35:03.639 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:35:03.639 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:35:03.639 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:35:03.639 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:35:03.639 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:35:03.639 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:35:03.639 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:35:03.639 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:03.639 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:03.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:03 smithi172 ceph-mon[65307]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:35:03.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:35:03.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:35:03.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:35:03.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:35:03.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:03.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:03.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:35:03.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:35:03.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:35:03.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:35:03.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:35:03.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:35:03.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:35:03.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:35:03.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:35:03.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:35:03.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:35:03.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:35:03.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:03.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:03.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:35:03.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:35:03.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:35:03.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:35:03.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:35:03.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:35:03.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:35:03.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:35:03.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:35:03.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:35:03.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:35:03.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:35:03.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:35:03.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:35:03.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:35:03.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:35:03.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:35:03.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:35:03.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:35:03.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:35:03.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:35:03.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:35:03.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:03.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:03.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:35:03.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:35:03.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:35:03.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:35:03.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:35:03.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:35:03.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:35:03.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:35:03.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:35:03.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:35:03.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:35:03.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:35:03.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:03.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:03.972 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:03 smithi172 ceph-mon[65307]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-04-22T15:35:03.972 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:03 smithi172 ceph-mon[65307]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:35:03.972 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:03 smithi172 ceph-mon[65307]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:35:03.973 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:03 smithi172 ceph-mon[65307]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:35:04.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:04 smithi145 ceph-mon[21899]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:35:04.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:35:04.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:35:04.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:35:04.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:35:04.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:04.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:04.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:35:04.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:35:04.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:35:04.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:35:04.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:35:04.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:35:04.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:35:04.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:35:04.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:35:04.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:35:04.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:35:04.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:35:04.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:04.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:04.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:35:04.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:35:04.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:35:04.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:35:04.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:35:04.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:35:04.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:35:04.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:35:04.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:35:04.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:35:04.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:35:04.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:35:04.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:35:04.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:35:04.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:35:04.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:35:04.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:35:04.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:35:04.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:35:04.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:35:04.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:35:04.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:35:04.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:04.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:04.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:35:04.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:35:04.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:35:04.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:35:04.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:35:04.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:35:04.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:35:04.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:35:04.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:35:04.877 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:04 smithi145 ceph-mon[26606]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:35:04.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:35:04.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:35:04.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:35:04.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:35:04.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:04.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:04.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:35:04.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:35:04.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:35:04.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:35:04.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:35:04.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:35:04.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:35:04.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:35:04.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:35:04.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:35:04.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:35:04.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:35:04.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:04.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:04.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:35:04.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:35:04.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:35:04.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:35:04.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:35:04.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:35:04.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:35:04.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:35:04.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:35:04.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:35:04.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:35:04.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:35:04.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:35:04.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:35:04.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:35:04.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:35:04.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:35:04.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:35:04.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:35:04.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:35:04.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:35:04.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:35:04.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:04.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:04.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:35:04.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:35:04.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:35:04.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:35:04.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:35:04.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:35:04.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:35:04.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:35:04.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:35:04.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:35:04.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:35:04.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:35:04.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:04.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:04.881 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:04 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:35:04.881 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:35:04.881 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:35:04.882 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:35:04.882 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:04.882 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:04.882 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:04 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:35:04.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:04 smithi172 ceph-mon[65307]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:35:04.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:35:04.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:35:04.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:35:04.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:35:04.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:04.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:04.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:35:04.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:35:04.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:35:04.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:35:04.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:35:04.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:35:04.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:35:04.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:35:04.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:35:04.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:35:04.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:35:04.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:35:04.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:04.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:04.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:35:04.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:35:04.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:35:04.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:35:04.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:35:04.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:35:04.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:35:04.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:35:04.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:35:04.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:35:04.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:35:04.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:35:04.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:35:04.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:35:04.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:35:04.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:35:04.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:35:04.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:35:04.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:35:04.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:35:04.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:35:04.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:35:04.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:04.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:04.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:35:04.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:35:04.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:35:04.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:35:04.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:35:04.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:35:04.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:35:04.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:35:04.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:35:04.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:35:04.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:35:04.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:35:04.973 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:04.973 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:04.973 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:04 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:35:05.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:05 smithi145 ceph-mon[21899]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 886 B/s rd, 0 op/s 2024-04-22T15:35:05.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:05 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:05.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:05 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:35:05] "GET /metrics HTTP/1.1" 200 38009 "" "Prometheus/2.43.0" 2024-04-22T15:35:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:05 smithi145 ceph-mon[26606]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 886 B/s rd, 0 op/s 2024-04-22T15:35:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:05 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:05.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:05 smithi172 ceph-mon[65307]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 886 B/s rd, 0 op/s 2024-04-22T15:35:05.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:05 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:07.770 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:07 smithi172 ceph-mon[65307]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:35:07.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:07 smithi145 ceph-mon[21899]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:35:07.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:07 smithi145 ceph-mon[26606]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:35:09.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:09 smithi145 ceph-mon[21899]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:35:09.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:09 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:35:09.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:09 smithi145 ceph-mon[26606]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:35:09.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:09 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:35:09.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:09 smithi172 ceph-mon[65307]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:35:09.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:09 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:35:11.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:11 smithi145 ceph-mon[21899]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:35:11.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:11 smithi145 ceph-mon[26606]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:35:11.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:11 smithi172 ceph-mon[65307]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:35:13.575 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch ps' 2024-04-22T15:35:13.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:13 smithi145 ceph-mon[21899]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:35:13.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:13 smithi145 ceph-mon[26606]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:35:13.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:13 smithi172 ceph-mon[65307]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:35:14.837 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:14 smithi145 ceph-mon[26606]: from='client.15699 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:35:14.837 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:14 smithi145 ceph-mon[21899]: from='client.15699 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:35:14.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:14 smithi172 ceph-mon[65307]: from='client.15699 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:35:15.476 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:35:15.477 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (2m) 2m ago 24m 16.0M - 0.25.0 c8568f914cd2 1c58ccf37ffd 2024-04-22T15:35:15.477 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (103s) 22s ago 24m 76.6M - 9.4.12 2bacad6d85d8 2f825d15ab10 2024-04-22T15:35:15.477 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 unknown 2m ago 23m 59.6M - 3.5 e1d6a67b021e 9704631ffa0e 2024-04-22T15:35:15.477 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443,9283,8765 running (3m) 22s ago 28m 453M - 19.0.0-3223-g5883d494 3d983c26e278 e7500ef71604 2024-04-22T15:35:15.477 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (13m) 2m ago 30m 505M - 19.0.0-3223-g5883d494 3d983c26e278 b2de97566cf4 2024-04-22T15:35:15.477 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (30m) 2m ago 30m 79.5M 2048M 17.2.0 e1d6a67b021e 28106a7bf16e 2024-04-22T15:35:15.477 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (26s) 22s ago 29m 21.7M 2048M 19.0.0-3223-g5883d494 3d983c26e278 0a360b08a6d6 2024-04-22T15:35:15.477 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (29m) 2m ago 29m 56.7M 2048M 17.2.0 e1d6a67b021e b71d9b9f8509 2024-04-22T15:35:15.477 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (3m) 2m ago 25m 8082k - 1.5.0 0da6a335fe13 8c75c26fec05 2024-04-22T15:35:15.477 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (2m) 22s ago 25m 15.4M - 1.5.0 0da6a335fe13 8fd2949e2eaf 2024-04-22T15:35:15.477 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (28m) 2m ago 28m 56.7M 2992M 17.2.0 e1d6a67b021e ea9cdc639916 2024-04-22T15:35:15.477 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (28m) 2m ago 28m 55.2M 2992M 17.2.0 e1d6a67b021e 3b9b37a8781f 2024-04-22T15:35:15.477 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (27m) 2m ago 27m 50.8M 2992M 17.2.0 e1d6a67b021e d9502dded8f8 2024-04-22T15:35:15.477 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (27m) 2m ago 27m 53.0M 2992M 17.2.0 e1d6a67b021e 8c72f30e54fc 2024-04-22T15:35:15.478 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (27m) 22s ago 27m 57.7M 3248M 17.2.0 e1d6a67b021e 80001f97ed35 2024-04-22T15:35:15.478 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (26m) 22s ago 26m 54.2M 3248M 17.2.0 e1d6a67b021e ec3ab1335b5c 2024-04-22T15:35:15.478 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (26m) 22s ago 26m 54.9M 3248M 17.2.0 e1d6a67b021e 45d475edf1ca 2024-04-22T15:35:15.478 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (25m) 22s ago 25m 52.9M 3248M 17.2.0 e1d6a67b021e cc9e1d9e7916 2024-04-22T15:35:15.478 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (2m) 22s ago 25m 63.9M - 2.43.0 a07b618ecd1d 51cf4df6e8d4 2024-04-22T15:35:15.478 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (23m) 2m ago 23m 78.0M - 17.2.0 e1d6a67b021e c90a636c947e 2024-04-22T15:35:15.478 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (23m) 22s ago 23m 76.1M - 17.2.0 e1d6a67b021e 20ac46e9bfda 2024-04-22T15:35:15.478 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:35:15] "GET /metrics HTTP/1.1" 200 38009 "" "Prometheus/2.43.0" 2024-04-22T15:35:15.835 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:15 smithi145 ceph-mon[26606]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:35:15.836 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:15 smithi145 ceph-mon[21899]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:35:15.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:15 smithi172 ceph-mon[65307]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:35:16.156 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 2'"'"'' 2024-04-22T15:35:16.733 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:16 smithi145 ceph-mon[26606]: from='client.15705 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:35:16.733 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:16 smithi145 ceph-mon[21899]: from='client.15705 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:35:16.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:16 smithi172 ceph-mon[65307]: from='client.15705 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:35:17.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:17 smithi145 ceph-mon[21899]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:35:17.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:17 smithi145 ceph-mon[26606]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:35:17.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:17 smithi172 ceph-mon[65307]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:35:18.344 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:35:18.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:18 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/3515015412' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:18.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:18 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/3515015412' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:18.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:18 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/3515015412' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:19.019 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch upgrade status' 2024-04-22T15:35:19.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:19 smithi145 ceph-mon[26606]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:35:19.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:19 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:35:19.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:19 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:35:19.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:19 smithi145 ceph-mon[21899]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:35:19.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:19 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:35:19.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:19 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:35:19.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:19 smithi172 ceph-mon[65307]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:35:19.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:19 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:35:19.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:19 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:35:21.151 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:35:21.151 INFO:teuthology.orchestra.run.smithi145.stdout: "target_image": null, 2024-04-22T15:35:21.151 INFO:teuthology.orchestra.run.smithi145.stdout: "in_progress": false, 2024-04-22T15:35:21.151 INFO:teuthology.orchestra.run.smithi145.stdout: "which": "", 2024-04-22T15:35:21.151 INFO:teuthology.orchestra.run.smithi145.stdout: "services_complete": [], 2024-04-22T15:35:21.152 INFO:teuthology.orchestra.run.smithi145.stdout: "progress": null, 2024-04-22T15:35:21.152 INFO:teuthology.orchestra.run.smithi145.stdout: "message": "", 2024-04-22T15:35:21.152 INFO:teuthology.orchestra.run.smithi145.stdout: "is_paused": false 2024-04-22T15:35:21.152 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:35:21.784 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph health detail' 2024-04-22T15:35:21.805 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:21 smithi145 ceph-mon[26606]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:35:21.805 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:21 smithi145 ceph-mon[21899]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:35:21.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:21 smithi172 ceph-mon[65307]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:35:22.784 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:22 smithi145 ceph-mon[26606]: from='client.15717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:35:22.784 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:22 smithi145 ceph-mon[21899]: from='client.15717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:35:22.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:22 smithi172 ceph-mon[65307]: from='client.15717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:35:23.633 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:23 smithi145 ceph-mon[26606]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:35:23.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:23 smithi145 ceph-mon[21899]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:35:23.903 INFO:teuthology.orchestra.run.smithi145.stdout:HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:35:23.904 INFO:teuthology.orchestra.run.smithi145.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:35:23.904 INFO:teuthology.orchestra.run.smithi145.stdout: daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state 2024-04-22T15:35:23.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:23 smithi172 ceph-mon[65307]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:35:24.517 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '"'"'{print $2}'"'"')' 2024-04-22T15:35:24.863 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:24 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/2265375405' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:35:24.863 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:24 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/2265375405' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:35:24.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:24 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/2265375405' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:35:25.558 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:35:25] "GET /metrics HTTP/1.1" 200 38011 "" "Prometheus/2.43.0" 2024-04-22T15:35:25.819 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:25 smithi145 ceph-mon[21899]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:35:25.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:25 smithi145 ceph-mon[26606]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:35:25.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:25 smithi172 ceph-mon[65307]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:35:27.848 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:27 smithi145 ceph-mon[21899]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:35:27.849 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:27 smithi145 ceph-mon[21899]: from='client.15729 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:35:27.849 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:27 smithi145 ceph-mon[26606]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:35:27.849 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:27 smithi145 ceph-mon[26606]: from='client.15729 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:35:27.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:27 smithi172 ceph-mon[65307]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:35:27.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:27 smithi172 ceph-mon[65307]: from='client.15729 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:35:28.778 INFO:teuthology.orchestra.run.smithi145.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:35:28.859 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:28 smithi145 ceph-mon[26606]: from='client.25657 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "daemon_types": "mon", "hosts": "smithi145", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:35:28.860 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:28 smithi145 ceph-mon[21899]: from='client.25657 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "daemon_types": "mon", "hosts": "smithi145", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:35:28.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:28 smithi172 ceph-mon[65307]: from='client.25657 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "daemon_types": "mon", "hosts": "smithi145", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:35:29.459 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-22T15:35:29.736 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:29 smithi145 ceph-mon[26606]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:35:29.736 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:29 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:35:29.736 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:29 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:29.736 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:29 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:35:29.736 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:29 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:35:29.736 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:29 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:35:29.736 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:29 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:29.736 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:29 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:35:29.737 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:29 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:35:29.737 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:29 smithi145 ceph-mon[21899]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:35:29.737 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:29 smithi145 ceph-mon[21899]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:35:29.737 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:29 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:29.737 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:29 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:35:29.737 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:29 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:35:29.737 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:29 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:35:29.737 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:29 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:29.737 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:29 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:35:29.737 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:29 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:35:29.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:29 smithi172 ceph-mon[65307]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:35:29.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:29 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:35:29.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:29 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:29.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:29 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:35:29.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:29 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:35:29.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:29 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:35:29.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:29 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:29.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:29 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:35:29.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:29 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:35:30.836 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:30 smithi145 ceph-mon[26606]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:35:30.836 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:30 smithi145 ceph-mon[26606]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 918 B/s rd, 0 op/s 2024-04-22T15:35:30.837 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:30 smithi145 ceph-mon[26606]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:35:30.837 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:30 smithi145 ceph-mon[26606]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:35:30.837 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:30 smithi145 ceph-mon[26606]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:35:30.837 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:30 smithi145 ceph-mon[21899]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:35:30.837 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:30 smithi145 ceph-mon[21899]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 918 B/s rd, 0 op/s 2024-04-22T15:35:30.837 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:30 smithi145 ceph-mon[21899]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:35:30.837 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:30 smithi145 ceph-mon[21899]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:35:30.837 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:30 smithi145 ceph-mon[21899]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:35:30.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:30 smithi172 ceph-mon[65307]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:35:30.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:30 smithi172 ceph-mon[65307]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 918 B/s rd, 0 op/s 2024-04-22T15:35:30.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:30 smithi172 ceph-mon[65307]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:35:30.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:30 smithi172 ceph-mon[65307]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:35:30.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:30 smithi172 ceph-mon[65307]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:35:32.066 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:35:31.895+0000 7f42a94bd700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:35:32.066 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:35:32.066 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:35:32.066 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:35:32.066 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:35:32.066 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:32.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:32.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:35:32.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:35:32.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:35:32.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:35:32.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:35:32.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:35:32.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:35:32.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:35:32.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:35:32.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:35:32.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:35:32.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:35:32.068 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:32.068 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:32.068 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:35:32.068 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:35:32.068 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:35:32.068 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:35:32.068 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:35:32.068 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:35:32.068 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:35:32.068 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:35:32.069 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:35:32.069 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:35:32.069 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:35:32.069 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:35:32.069 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:35:32.069 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:35:32.069 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:35:32.069 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:35:32.069 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:35:32.069 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:35:32.069 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:35:32.069 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:35:32.069 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:35:32.069 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:35:32.069 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:32.070 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:32.070 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:35:32.070 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:35:32.070 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:35:32.070 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:35:32.070 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:35:32.070 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:35:32.070 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:35:32.070 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:35:32.070 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:35:32.070 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:35:32.070 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:35:32.070 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:35:32.071 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:32.071 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:32.558 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:35:32.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:32 smithi145 ceph-mon[26606]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:35:32.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:32 smithi145 ceph-mon[21899]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:35:32.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:32 smithi172 ceph-mon[65307]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:35:33.230 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:35:33.230 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (2m) 2m ago 24m 16.0M - 0.25.0 c8568f914cd2 1c58ccf37ffd 2024-04-22T15:35:33.230 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (2m) 39s ago 24m 76.6M - 9.4.12 2bacad6d85d8 2f825d15ab10 2024-04-22T15:35:33.230 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 unknown 2m ago 23m 59.6M - 3.5 e1d6a67b021e 9704631ffa0e 2024-04-22T15:35:33.230 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443,9283,8765 running (3m) 39s ago 29m 453M - 19.0.0-3223-g5883d494 3d983c26e278 e7500ef71604 2024-04-22T15:35:33.230 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (13m) 2m ago 30m 505M - 19.0.0-3223-g5883d494 3d983c26e278 b2de97566cf4 2024-04-22T15:35:33.230 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (30m) 2m ago 30m 79.5M 2048M 17.2.0 e1d6a67b021e 28106a7bf16e 2024-04-22T15:35:33.230 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (44s) 39s ago 29m 21.7M 2048M 19.0.0-3223-g5883d494 3d983c26e278 0a360b08a6d6 2024-04-22T15:35:33.230 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (29m) 2m ago 29m 56.7M 2048M 17.2.0 e1d6a67b021e b71d9b9f8509 2024-04-22T15:35:33.230 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (3m) 2m ago 25m 8082k - 1.5.0 0da6a335fe13 8c75c26fec05 2024-04-22T15:35:33.230 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (3m) 39s ago 25m 15.4M - 1.5.0 0da6a335fe13 8fd2949e2eaf 2024-04-22T15:35:33.231 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (28m) 2m ago 28m 56.7M 2992M 17.2.0 e1d6a67b021e ea9cdc639916 2024-04-22T15:35:33.231 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (28m) 2m ago 28m 55.2M 2992M 17.2.0 e1d6a67b021e 3b9b37a8781f 2024-04-22T15:35:33.231 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (28m) 2m ago 28m 50.8M 2992M 17.2.0 e1d6a67b021e d9502dded8f8 2024-04-22T15:35:33.231 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (27m) 2m ago 27m 53.0M 2992M 17.2.0 e1d6a67b021e 8c72f30e54fc 2024-04-22T15:35:33.231 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (27m) 39s ago 27m 57.7M 3248M 17.2.0 e1d6a67b021e 80001f97ed35 2024-04-22T15:35:33.231 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (27m) 39s ago 27m 54.2M 3248M 17.2.0 e1d6a67b021e ec3ab1335b5c 2024-04-22T15:35:33.231 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (26m) 39s ago 26m 54.9M 3248M 17.2.0 e1d6a67b021e 45d475edf1ca 2024-04-22T15:35:33.231 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (26m) 39s ago 26m 52.9M 3248M 17.2.0 e1d6a67b021e cc9e1d9e7916 2024-04-22T15:35:33.231 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (2m) 39s ago 25m 63.9M - 2.43.0 a07b618ecd1d 51cf4df6e8d4 2024-04-22T15:35:33.231 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (23m) 2m ago 23m 78.0M - 17.2.0 e1d6a67b021e c90a636c947e 2024-04-22T15:35:33.231 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (23m) 39s ago 23m 76.1M - 17.2.0 e1d6a67b021e 20ac46e9bfda 2024-04-22T15:35:33.644 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:35:33.644 INFO:teuthology.orchestra.run.smithi145.stdout: "mon": { 2024-04-22T15:35:33.644 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-04-22T15:35:33.645 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 1 2024-04-22T15:35:33.645 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:35:33.645 INFO:teuthology.orchestra.run.smithi145.stdout: "mgr": { 2024-04-22T15:35:33.645 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:35:33.645 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:35:33.645 INFO:teuthology.orchestra.run.smithi145.stdout: "osd": { 2024-04-22T15:35:33.645 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T15:35:33.645 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:35:33.645 INFO:teuthology.orchestra.run.smithi145.stdout: "mds": {}, 2024-04-22T15:35:33.645 INFO:teuthology.orchestra.run.smithi145.stdout: "rgw": { 2024-04-22T15:35:33.645 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:35:33.645 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:35:33.645 INFO:teuthology.orchestra.run.smithi145.stdout: "overall": { 2024-04-22T15:35:33.645 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-04-22T15:35:33.645 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 3 2024-04-22T15:35:33.645 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:35:33.645 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:35:33.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:33 smithi145 ceph-mon[26606]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:35:33.873 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:35:33.873 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:35:33.873 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:35:33.873 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:35:33.873 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:33.873 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:33.873 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:35:33.873 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:35:33.873 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:35:33.873 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:35:33.873 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:35:33.874 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:35:33.874 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:35:33.874 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:35:33.874 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:35:33.874 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:35:33.874 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:35:33.874 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:35:33.874 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:33.874 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:33.874 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:35:33.874 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:35:33.874 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:35:33.874 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:35:33.874 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:35:33.875 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:35:33.875 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:35:33.875 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:35:33.875 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:35:33.875 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:35:33.875 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:35:33.875 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:35:33.875 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:35:33.875 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:35:33.875 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:35:33.875 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:35:33.875 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:35:33.875 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:35:33.875 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:35:33.875 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:35:33.875 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:35:33.876 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:35:33.876 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:33.876 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:33.876 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:35:33.876 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:35:33.876 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:35:33.876 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:35:33.876 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:35:33.876 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:35:33.876 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:35:33.876 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:35:33.876 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:35:33.877 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:33 smithi145 ceph-mon[21899]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:35:33.877 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:35:33.877 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:35:33.877 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:35:33.877 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:35:33.877 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:33.877 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:33.877 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:35:33.877 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:35:33.877 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:35:33.877 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:35:33.877 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:35:33.877 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:35:33.878 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:35:33.878 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:35:33.878 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:35:33.878 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:35:33.878 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:35:33.878 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:35:33.878 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:33.878 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:33.878 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:35:33.878 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:35:33.878 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:35:33.878 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:35:33.878 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:35:33.878 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:35:33.878 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:35:33.879 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:35:33.879 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:35:33.879 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:35:33.879 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:35:33.879 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:35:33.879 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:35:33.879 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:35:33.879 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:35:33.879 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:35:33.879 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:35:33.879 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:35:33.879 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:35:33.879 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:35:33.879 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:35:33.879 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:35:33.879 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:33.880 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:33.880 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:35:33.880 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:35:33.880 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:35:33.880 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:35:33.880 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:35:33.880 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:35:33.880 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:35:33.880 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:35:33.880 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:35:33.880 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:35:33.880 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:35:33.880 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:35:33.880 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:33.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:33.881 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:33 smithi145 ceph-mon[21899]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:35:33.881 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:33 smithi145 ceph-mon[21899]: from='client.25663 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:35:33.881 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:35:33.881 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:35:33.881 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:35:33.881 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:33.881 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:33.881 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:33 smithi145 ceph-mon[26606]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:35:33.881 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:33 smithi145 ceph-mon[26606]: from='client.25663 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:35:33.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:33 smithi172 ceph-mon[65307]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:35:33.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:35:33.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:35:33.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:35:33.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:35:33.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:33.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:33.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:35:33.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:35:33.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:35:33.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:35:33.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:35:33.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:35:33.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:35:33.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:35:33.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:35:33.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:35:33.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:35:33.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:35:33.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:33.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:33.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:35:33.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:35:33.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:35:33.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:35:33.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:35:33.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:35:33.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:35:33.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:35:33.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:35:33.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:35:33.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:35:33.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:35:33.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:35:33.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:35:33.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:35:33.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:35:33.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:35:33.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:35:33.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:35:33.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:35:33.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:35:33.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:35:33.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:33.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:33.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:35:33.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:35:33.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:35:33.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:35:33.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:35:33.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:35:33.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:35:33.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:35:33.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:35:33.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:35:33.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:35:33.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:35:33.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:35:33.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:35:33.972 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:33 smithi172 ceph-mon[65307]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:35:33.973 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:33 smithi172 ceph-mon[65307]: from='client.25663 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:35:33.983 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:35:33.983 INFO:teuthology.orchestra.run.smithi145.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:35:33.983 INFO:teuthology.orchestra.run.smithi145.stdout: "in_progress": true, 2024-04-22T15:35:33.983 INFO:teuthology.orchestra.run.smithi145.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi145", 2024-04-22T15:35:33.983 INFO:teuthology.orchestra.run.smithi145.stdout: "services_complete": [], 2024-04-22T15:35:33.983 INFO:teuthology.orchestra.run.smithi145.stdout: "progress": "0/2 daemons upgraded", 2024-04-22T15:35:33.983 INFO:teuthology.orchestra.run.smithi145.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa image", 2024-04-22T15:35:33.983 INFO:teuthology.orchestra.run.smithi145.stdout: "is_paused": false 2024-04-22T15:35:33.984 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:35:34.863 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:34 smithi145 ceph-mon[26606]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:35:34.863 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:34 smithi145 ceph-mon[26606]: from='client.15747 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:35:34.863 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:34 smithi145 ceph-mon[26606]: from='client.25675 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:35:34.863 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:34 smithi145 ceph-mon[26606]: from='client.? 172.21.15.145:0/20835932' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:34.863 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:34 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:34.863 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:34 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:35:34.863 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:34 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:34.863 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:34 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:34.863 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:34 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-22T15:35:34.863 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:34 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-22T15:35:34.863 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:34 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:35:34.864 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:34 smithi145 ceph-mon[21899]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:35:34.864 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:34 smithi145 ceph-mon[21899]: from='client.15747 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:35:34.864 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:34 smithi145 ceph-mon[21899]: from='client.25675 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:35:34.864 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:34 smithi145 ceph-mon[21899]: from='client.? 172.21.15.145:0/20835932' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:34.864 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:34 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:34.864 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:34 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:35:34.864 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:34 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:34.864 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:34 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:34.864 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:34 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-22T15:35:34.864 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:34 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-22T15:35:34.864 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:34 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:35:34.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:34 smithi172 ceph-mon[65307]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:35:34.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:34 smithi172 ceph-mon[65307]: from='client.15747 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:35:34.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:34 smithi172 ceph-mon[65307]: from='client.25675 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:35:34.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:34 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/20835932' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:34.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:34 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:34.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:34 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:35:34.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:34 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:35:34.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:34 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:34.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:34 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-22T15:35:34.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:34 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-22T15:35:34.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:34 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:35:35.550 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:35:35] "GET /metrics HTTP/1.1" 200 38013 "" "Prometheus/2.43.0" 2024-04-22T15:35:35.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:35 smithi145 ceph-mon[26606]: Upgrade: Target is version 19.0.0-3223-g5883d494 (squid) 2024-04-22T15:35:35.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:35 smithi145 ceph-mon[26606]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, digests ['quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb'] 2024-04-22T15:35:35.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:35 smithi145 ceph-mon[26606]: Upgrade: Setting container_image for all mgr 2024-04-22T15:35:35.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:35 smithi145 ceph-mon[26606]: from='client.25687 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:35:35.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:35 smithi145 ceph-mon[26606]: Upgrade: It appears safe to stop mon.a 2024-04-22T15:35:35.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:35 smithi145 ceph-mon[21899]: Upgrade: Target is version 19.0.0-3223-g5883d494 (squid) 2024-04-22T15:35:35.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:35 smithi145 ceph-mon[21899]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, digests ['quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb'] 2024-04-22T15:35:35.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:35 smithi145 ceph-mon[21899]: Upgrade: Setting container_image for all mgr 2024-04-22T15:35:35.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:35 smithi145 ceph-mon[21899]: from='client.25687 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:35:35.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:35 smithi145 ceph-mon[21899]: Upgrade: It appears safe to stop mon.a 2024-04-22T15:35:35.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:35 smithi172 ceph-mon[65307]: Upgrade: Target is version 19.0.0-3223-g5883d494 (squid) 2024-04-22T15:35:35.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:35 smithi172 ceph-mon[65307]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, digests ['quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb'] 2024-04-22T15:35:35.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:35 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all mgr 2024-04-22T15:35:35.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:35 smithi172 ceph-mon[65307]: from='client.25687 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:35:35.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:35 smithi172 ceph-mon[65307]: Upgrade: It appears safe to stop mon.a 2024-04-22T15:35:36.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:36 smithi145 ceph-mon[21899]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 559 B/s rd, 0 op/s 2024-04-22T15:35:36.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:36 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:36.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:36 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:35:36.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:36 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:35:36.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:36 smithi145 ceph-mon[21899]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:35:36.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:36 smithi145 ceph-mon[26606]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 559 B/s rd, 0 op/s 2024-04-22T15:35:36.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:36 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:36.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:36 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:35:36.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:36 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:35:36.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:36 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:35:36.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:36 smithi172 ceph-mon[65307]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 559 B/s rd, 0 op/s 2024-04-22T15:35:36.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:36 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' 2024-04-22T15:35:36.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:36 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:35:36.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:36 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:35:36.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:36 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:35:37.697 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:37 smithi145 ceph-mon[26606]: Upgrade: Updating mon.a 2024-04-22T15:35:37.697 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:37 smithi145 ceph-mon[26606]: Deploying daemon mon.a on smithi145 2024-04-22T15:35:37.698 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:37 smithi145 systemd[1]: Stopping Ceph mon.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:35:37.698 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:37 smithi145 ceph-mon[21899]: Upgrade: Updating mon.a 2024-04-22T15:35:37.698 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:37 smithi145 ceph-mon[21899]: Deploying daemon mon.a on smithi145 2024-04-22T15:35:37.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:37 smithi172 ceph-mon[65307]: Upgrade: Updating mon.a 2024-04-22T15:35:37.869 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:37 smithi172 ceph-mon[65307]: Deploying daemon mon.a on smithi145 2024-04-22T15:35:38.120 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-a[21895]: 2024-04-22T15:35:37.693+0000 7f44660f0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:35:38.121 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-a[21895]: 2024-04-22T15:35:37.693+0000 7f44660f0700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-04-22T15:35:38.121 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:37 smithi145 podman[82346]: 2024-04-22 15:35:37.744836925 +0000 UTC m=+0.175585767 container died 28106a7bf16eac08b626cb72a9b68da00df5d6be5a9f3071115b05e124b62ce6 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-a, vendor=Red Hat, Inc., GIT_CLEAN=True, io.buildah.version=1.19.8, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, vcs-type=git, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, architecture=x86_64) 2024-04-22T15:35:38.121 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:35:37] ENGINE Bus STOPPING 2024-04-22T15:35:38.121 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:35:37] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-22T15:35:38.121 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:35:37] ENGINE Bus STOPPED 2024-04-22T15:35:38.121 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:35:37] ENGINE Bus STARTING 2024-04-22T15:35:38.121 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:35:37] ENGINE Serving on http://:::9283 2024-04-22T15:35:38.121 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:35:37] ENGINE Bus STARTED 2024-04-22T15:35:38.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:38 smithi145 podman[82346]: 2024-04-22 15:35:38.118298752 +0000 UTC m=+0.549047591 container cleanup 28106a7bf16eac08b626cb72a9b68da00df5d6be5a9f3071115b05e124b62ce6 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-a, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , version=8, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, ceph=True, name=centos-stream, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, vendor=Red Hat, Inc., io.openshift.expose-services=) 2024-04-22T15:35:38.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:38 smithi145 bash[82346]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-a 2024-04-22T15:35:38.736 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:38 smithi145 podman[82370]: 2024-04-22 15:35:38.403208964 +0000 UTC m=+0.658838614 container remove 28106a7bf16eac08b626cb72a9b68da00df5d6be5a9f3071115b05e124b62ce6 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-a, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, vendor=Red Hat, Inc., RELEASE=HEAD, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, com.redhat.component=centos-stream-container, distribution-scope=public) 2024-04-22T15:35:38.736 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:38 smithi145 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mon.a.service: Deactivated successfully. 2024-04-22T15:35:38.736 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:38 smithi145 systemd[1]: Stopped Ceph mon.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:35:38.736 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:38 smithi145 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mon.a.service: Consumed 25.671s CPU time. 2024-04-22T15:35:39.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:38 smithi145 systemd[1]: Starting Ceph mon.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:35:39.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:38 smithi145 podman[82505]: 2024-04-22 15:35:38.853232501 +0000 UTC m=+0.017879196 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:35:39.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:38 smithi145 podman[82505]: 2024-04-22 15:35:38.954647278 +0000 UTC m=+0.119293976 container create 7bfdc11120f4affc98db5c05200905b159038265bb472095719837905fe1fb72 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-a, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-04-22T15:35:39.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 podman[82505]: 2024-04-22 15:35:39.19064027 +0000 UTC m=+0.355286965 container init 7bfdc11120f4affc98db5c05200905b159038265bb472095719837905fe1fb72 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-a, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416) 2024-04-22T15:35:39.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 podman[82505]: 2024-04-22 15:35:39.194579234 +0000 UTC m=+0.359225930 container start 7bfdc11120f4affc98db5c05200905b159038265bb472095719837905fe1fb72 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-a, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T15:35:39.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: set uid:gid to 167:167 (ceph:ceph) 2024-04-22T15:35:39.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev), process ceph-mon, pid 2 2024-04-22T15:35:39.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: pidfile_write: ignore empty --pid-file 2024-04-22T15:35:39.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: load: jerasure load: lrc 2024-04-22T15:35:39.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: RocksDB version: 7.9.2 2024-04-22T15:35:39.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Git sha 0 2024-04-22T15:35:39.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Compile date 2024-04-22 12:55:29 2024-04-22T15:35:39.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: DB SUMMARY 2024-04-22T15:35:39.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: DB Session ID: CFNRUO8R5S7MCXOY3HKS 2024-04-22T15:35:39.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: CURRENT file: CURRENT 2024-04-22T15:35:39.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: IDENTITY file: IDENTITY 2024-04-22T15:35:39.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: MANIFEST file: MANIFEST-000015 size: 3080 Bytes 2024-04-22T15:35:39.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000063.sst 2024-04-22T15:35:39.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000061.log size: 366852 ; 2024-04-22T15:35:39.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.error_if_exists: 0 2024-04-22T15:35:39.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.create_if_missing: 0 2024-04-22T15:35:39.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.paranoid_checks: 1 2024-04-22T15:35:39.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-22T15:35:39.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-22T15:35:39.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-22T15:35:39.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.env: 0x56209bddacc0 2024-04-22T15:35:39.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.fs: PosixFileSystem 2024-04-22T15:35:39.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.info_log: 0x56209d363be0 2024-04-22T15:35:39.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.max_file_opening_threads: 16 2024-04-22T15:35:39.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.statistics: (nil) 2024-04-22T15:35:39.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.use_fsync: 0 2024-04-22T15:35:39.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.max_log_file_size: 0 2024-04-22T15:35:39.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-22T15:35:39.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-22T15:35:39.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.keep_log_file_num: 1000 2024-04-22T15:35:39.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.recycle_log_file_num: 0 2024-04-22T15:35:39.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.allow_fallocate: 1 2024-04-22T15:35:39.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.allow_mmap_reads: 0 2024-04-22T15:35:39.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.allow_mmap_writes: 0 2024-04-22T15:35:39.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.use_direct_reads: 0 2024-04-22T15:35:39.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-22T15:35:39.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.create_missing_column_families: 0 2024-04-22T15:35:39.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.db_log_dir: 2024-04-22T15:35:39.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.wal_dir: 2024-04-22T15:35:39.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-22T15:35:39.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-22T15:35:39.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-22T15:35:39.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-22T15:35:39.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-22T15:35:39.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-22T15:35:39.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.advise_random_on_open: 1 2024-04-22T15:35:39.377 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.db_write_buffer_size: 0 2024-04-22T15:35:39.377 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.write_buffer_manager: 0x56209d2f7220 2024-04-22T15:35:39.377 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-22T15:35:39.377 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-22T15:35:39.377 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-22T15:35:39.377 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.rate_limiter: (nil) 2024-04-22T15:35:39.377 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-22T15:35:39.377 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.wal_recovery_mode: 2 2024-04-22T15:35:39.377 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.enable_thread_tracking: 0 2024-04-22T15:35:39.377 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.enable_pipelined_write: 0 2024-04-22T15:35:39.377 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.unordered_write: 0 2024-04-22T15:35:39.377 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-22T15:35:39.377 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-22T15:35:39.377 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-22T15:35:39.377 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-22T15:35:39.378 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.row_cache: None 2024-04-22T15:35:39.378 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.wal_filter: None 2024-04-22T15:35:39.379 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-22T15:35:39.379 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.allow_ingest_behind: 0 2024-04-22T15:35:39.379 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.two_write_queues: 0 2024-04-22T15:35:39.379 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.manual_wal_flush: 0 2024-04-22T15:35:39.379 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.wal_compression: 0 2024-04-22T15:35:39.379 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.atomic_flush: 0 2024-04-22T15:35:39.379 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-22T15:35:39.379 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-22T15:35:39.379 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-22T15:35:39.379 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.log_readahead_size: 0 2024-04-22T15:35:39.379 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-22T15:35:39.379 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.best_efforts_recovery: 0 2024-04-22T15:35:39.380 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-22T15:35:39.380 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-22T15:35:39.380 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.allow_data_in_errors: 0 2024-04-22T15:35:39.380 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.db_host_id: __hostname__ 2024-04-22T15:35:39.380 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-22T15:35:39.380 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.max_background_jobs: 2 2024-04-22T15:35:39.380 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.max_background_compactions: -1 2024-04-22T15:35:39.380 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.max_subcompactions: 1 2024-04-22T15:35:39.380 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-22T15:35:39.380 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-22T15:35:39.381 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-22T15:35:39.381 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.max_total_wal_size: 0 2024-04-22T15:35:39.381 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-22T15:35:39.381 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-22T15:35:39.381 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-22T15:35:39.381 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-22T15:35:39.381 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.max_open_files: -1 2024-04-22T15:35:39.381 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.bytes_per_sync: 0 2024-04-22T15:35:39.381 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-22T15:35:39.381 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-22T15:35:39.381 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.compaction_readahead_size: 0 2024-04-22T15:35:39.381 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.max_background_flushes: -1 2024-04-22T15:35:39.381 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Compression algorithms supported: 2024-04-22T15:35:39.381 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-22T15:35:39.382 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: kZSTD supported: 0 2024-04-22T15:35:39.382 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: kXpressCompression supported: 0 2024-04-22T15:35:39.382 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: kLZ4HCCompression supported: 1 2024-04-22T15:35:39.382 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: kZlibCompression supported: 1 2024-04-22T15:35:39.382 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: kSnappyCompression supported: 1 2024-04-22T15:35:39.382 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: kLZ4Compression supported: 1 2024-04-22T15:35:39.382 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: kBZip2Compression supported: 0 2024-04-22T15:35:39.382 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-22T15:35:39.382 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-22T15:35:39.382 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-04-22T15:35:39.382 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-22T15:35:39.382 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-22T15:35:39.382 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.merge_operator: 2024-04-22T15:35:39.382 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.compaction_filter: None 2024-04-22T15:35:39.383 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.compaction_filter_factory: None 2024-04-22T15:35:39.383 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.sst_partitioner_factory: None 2024-04-22T15:35:39.383 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-22T15:35:39.383 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-22T15:35:39.383 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56209d363dc0) 2024-04-22T15:35:39.383 INFO:journalctl@ceph.mon.a.smithi145.stdout: cache_index_and_filter_blocks: 1 2024-04-22T15:35:39.383 INFO:journalctl@ceph.mon.a.smithi145.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-22T15:35:39.383 INFO:journalctl@ceph.mon.a.smithi145.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-22T15:35:39.383 INFO:journalctl@ceph.mon.a.smithi145.stdout: pin_top_level_index_and_filter: 1 2024-04-22T15:35:39.383 INFO:journalctl@ceph.mon.a.smithi145.stdout: index_type: 0 2024-04-22T15:35:39.383 INFO:journalctl@ceph.mon.a.smithi145.stdout: data_block_index_type: 0 2024-04-22T15:35:39.383 INFO:journalctl@ceph.mon.a.smithi145.stdout: index_shortening: 1 2024-04-22T15:35:39.383 INFO:journalctl@ceph.mon.a.smithi145.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-22T15:35:39.383 INFO:journalctl@ceph.mon.a.smithi145.stdout: checksum: 4 2024-04-22T15:35:39.383 INFO:journalctl@ceph.mon.a.smithi145.stdout: no_block_cache: 0 2024-04-22T15:35:39.384 INFO:journalctl@ceph.mon.a.smithi145.stdout: block_cache: 0x56209d3291f0 2024-04-22T15:35:39.384 INFO:journalctl@ceph.mon.a.smithi145.stdout: block_cache_name: BinnedLRUCache 2024-04-22T15:35:39.384 INFO:journalctl@ceph.mon.a.smithi145.stdout: block_cache_options: 2024-04-22T15:35:39.384 INFO:journalctl@ceph.mon.a.smithi145.stdout: capacity : 536870912 2024-04-22T15:35:39.384 INFO:journalctl@ceph.mon.a.smithi145.stdout: num_shard_bits : 4 2024-04-22T15:35:39.384 INFO:journalctl@ceph.mon.a.smithi145.stdout: strict_capacity_limit : 0 2024-04-22T15:35:39.384 INFO:journalctl@ceph.mon.a.smithi145.stdout: high_pri_pool_ratio: 0.000 2024-04-22T15:35:39.384 INFO:journalctl@ceph.mon.a.smithi145.stdout: block_cache_compressed: (nil) 2024-04-22T15:35:39.384 INFO:journalctl@ceph.mon.a.smithi145.stdout: persistent_cache: (nil) 2024-04-22T15:35:39.384 INFO:journalctl@ceph.mon.a.smithi145.stdout: block_size: 4096 2024-04-22T15:35:39.384 INFO:journalctl@ceph.mon.a.smithi145.stdout: block_size_deviation: 10 2024-04-22T15:35:39.384 INFO:journalctl@ceph.mon.a.smithi145.stdout: block_restart_interval: 16 2024-04-22T15:35:39.384 INFO:journalctl@ceph.mon.a.smithi145.stdout: index_block_restart_interval: 1 2024-04-22T15:35:39.384 INFO:journalctl@ceph.mon.a.smithi145.stdout: metadata_block_size: 4096 2024-04-22T15:35:39.384 INFO:journalctl@ceph.mon.a.smithi145.stdout: partition_filters: 0 2024-04-22T15:35:39.385 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_delta_encoding: 1 2024-04-22T15:35:39.385 INFO:journalctl@ceph.mon.a.smithi145.stdout: filter_policy: bloomfilter 2024-04-22T15:35:39.385 INFO:journalctl@ceph.mon.a.smithi145.stdout: whole_key_filtering: 1 2024-04-22T15:35:39.385 INFO:journalctl@ceph.mon.a.smithi145.stdout: verify_compression: 0 2024-04-22T15:35:39.385 INFO:journalctl@ceph.mon.a.smithi145.stdout: read_amp_bytes_per_bit: 0 2024-04-22T15:35:39.385 INFO:journalctl@ceph.mon.a.smithi145.stdout: format_version: 5 2024-04-22T15:35:39.385 INFO:journalctl@ceph.mon.a.smithi145.stdout: enable_index_compression: 1 2024-04-22T15:35:39.385 INFO:journalctl@ceph.mon.a.smithi145.stdout: block_align: 0 2024-04-22T15:35:39.385 INFO:journalctl@ceph.mon.a.smithi145.stdout: max_auto_readahead_size: 262144 2024-04-22T15:35:39.385 INFO:journalctl@ceph.mon.a.smithi145.stdout: prepopulate_block_cache: 0 2024-04-22T15:35:39.385 INFO:journalctl@ceph.mon.a.smithi145.stdout: initial_auto_readahead_size: 8192 2024-04-22T15:35:39.385 INFO:journalctl@ceph.mon.a.smithi145.stdout: num_file_reads_for_auto_readahead: 2 2024-04-22T15:35:39.386 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.write_buffer_size: 33554432 2024-04-22T15:35:39.386 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.max_write_buffer_number: 2 2024-04-22T15:35:39.386 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.compression: NoCompression 2024-04-22T15:35:39.386 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.bottommost_compression: Disabled 2024-04-22T15:35:39.386 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.prefix_extractor: nullptr 2024-04-22T15:35:39.386 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-22T15:35:39.386 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.num_levels: 7 2024-04-22T15:35:39.386 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-22T15:35:39.386 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-22T15:35:39.386 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-22T15:35:39.386 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-22T15:35:39.386 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-22T15:35:39.386 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-22T15:35:39.386 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-22T15:35:39.387 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-22T15:35:39.387 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-22T15:35:39.387 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-22T15:35:39.387 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-22T15:35:39.387 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-22T15:35:39.387 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-22T15:35:39.387 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.compression_opts.level: 32767 2024-04-22T15:35:39.387 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.compression_opts.strategy: 0 2024-04-22T15:35:39.387 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-22T15:35:39.387 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-22T15:35:39.387 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-22T15:35:39.387 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-22T15:35:39.387 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.compression_opts.enabled: false 2024-04-22T15:35:39.387 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-22T15:35:39.388 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-22T15:35:39.388 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-22T15:35:39.388 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-22T15:35:39.388 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.target_file_size_base: 67108864 2024-04-22T15:35:39.388 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-22T15:35:39.388 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-22T15:35:39.388 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-22T15:35:39.388 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-22T15:35:39.388 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-22T15:35:39.388 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-22T15:35:39.388 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-22T15:35:39.388 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-22T15:35:39.388 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-22T15:35:39.388 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-22T15:35:39.388 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-22T15:35:39.388 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-22T15:35:39.389 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-22T15:35:39.389 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-22T15:35:39.389 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.arena_block_size: 1048576 2024-04-22T15:35:39.389 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-22T15:35:39.389 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-22T15:35:39.389 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.disable_auto_compactions: 0 2024-04-22T15:35:39.389 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-22T15:35:39.389 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-22T15:35:39.389 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-22T15:35:39.389 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-22T15:35:39.389 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-22T15:35:39.389 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-22T15:35:39.389 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-22T15:35:39.389 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-22T15:35:39.389 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-22T15:35:39.390 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-22T15:35:39.390 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-04-22T15:35:39.390 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.inplace_update_support: 0 2024-04-22T15:35:39.390 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-22T15:35:39.390 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-22T15:35:39.390 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-22T15:35:39.390 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-22T15:35:39.390 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.bloom_locality: 0 2024-04-22T15:35:39.390 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.max_successive_merges: 0 2024-04-22T15:35:39.390 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-22T15:35:39.390 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.paranoid_file_checks: 0 2024-04-22T15:35:39.390 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.force_consistency_checks: 1 2024-04-22T15:35:39.390 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.report_bg_io_stats: 0 2024-04-22T15:35:39.390 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.ttl: 2592000 2024-04-22T15:35:39.390 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-22T15:35:39.391 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-22T15:35:39.391 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-22T15:35:39.391 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.enable_blob_files: false 2024-04-22T15:35:39.391 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.min_blob_size: 0 2024-04-22T15:35:39.391 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.blob_file_size: 268435456 2024-04-22T15:35:39.391 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-22T15:35:39.391 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-22T15:35:39.391 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-22T15:35:39.391 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-22T15:35:39.392 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-22T15:35:39.392 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.blob_file_starting_level: 0 2024-04-22T15:35:39.392 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-22T15:35:39.392 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 63.sst 2024-04-22T15:35:39.392 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-22T15:35:39.392 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 65, last_sequence is 29313, log_number is 61,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-22T15:35:39.392 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 61 2024-04-22T15:35:39.392 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: ee7a0575-f010-4b3a-8c13-6d12ef8c1f92 2024-04-22T15:35:39.392 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713800139224462, "job": 1, "event": "recovery_started", "wal_files": [61]} 2024-04-22T15:35:39.392 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #61 mode 2 2024-04-22T15:35:39.392 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713800139227423, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 66, "file_size": 357601, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 29314, "largest_seqno": 29687, "table_properties": {"data_size": 355239, "index_size": 874, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 453, "raw_key_size": 3923, "raw_average_key_size": 23, "raw_value_size": 351766, "raw_average_value_size": 2106, "num_data_blocks": 39, "num_entries": 167, "num_filter_entries": 167, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1713800139, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "ee7a0575-f010-4b3a-8c13-6d12ef8c1f92", "db_session_id": "CFNRUO8R5S7MCXOY3HKS", "orig_file_number": 66, "seqno_to_time_mapping": "N/A"}} 2024-04-22T15:35:39.392 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713800139227505, "job": 1, "event": "recovery_finished"} 2024-04-22T15:35:39.392 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: [db/version_set.cc:5047] Creating manifest 68 2024-04-22T15:35:39.392 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-22T15:35:39.392 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000061.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-22T15:35:39.393 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x56209d43e000 2024-04-22T15:35:39.393 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: DB pointer 0x56209d428000 2024-04-22T15:35:39.393 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-22T15:35:39.393 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-22T15:35:39.393 INFO:journalctl@ceph.mon.a.smithi145.stdout: ** DB Stats ** 2024-04-22T15:35:39.393 INFO:journalctl@ceph.mon.a.smithi145.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T15:35:39.393 INFO:journalctl@ceph.mon.a.smithi145.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-22T15:35:39.393 INFO:journalctl@ceph.mon.a.smithi145.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-22T15:35:39.393 INFO:journalctl@ceph.mon.a.smithi145.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T15:35:39.393 INFO:journalctl@ceph.mon.a.smithi145.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-22T15:35:39.393 INFO:journalctl@ceph.mon.a.smithi145.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-22T15:35:39.393 INFO:journalctl@ceph.mon.a.smithi145.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T15:35:39.393 INFO:journalctl@ceph.mon.a.smithi145.stdout: 2024-04-22T15:35:39.393 INFO:journalctl@ceph.mon.a.smithi145.stdout: ** Compaction Stats [default] ** 2024-04-22T15:35:39.393 INFO:journalctl@ceph.mon.a.smithi145.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-22T15:35:39.394 INFO:journalctl@ceph.mon.a.smithi145.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-22T15:35:39.394 INFO:journalctl@ceph.mon.a.smithi145.stdout: L0 1/0 349.22 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 190.6 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-04-22T15:35:39.394 INFO:journalctl@ceph.mon.a.smithi145.stdout: L6 1/0 10.56 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-22T15:35:39.394 INFO:journalctl@ceph.mon.a.smithi145.stdout: Sum 2/0 10.90 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 190.6 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-04-22T15:35:39.394 INFO:journalctl@ceph.mon.a.smithi145.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 190.6 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-04-22T15:35:39.394 INFO:journalctl@ceph.mon.a.smithi145.stdout: 2024-04-22T15:35:39.394 INFO:journalctl@ceph.mon.a.smithi145.stdout: ** Compaction Stats [default] ** 2024-04-22T15:35:39.394 INFO:journalctl@ceph.mon.a.smithi145.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-22T15:35:39.394 INFO:journalctl@ceph.mon.a.smithi145.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T15:35:39.394 INFO:journalctl@ceph.mon.a.smithi145.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 190.6 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-04-22T15:35:39.394 INFO:journalctl@ceph.mon.a.smithi145.stdout: 2024-04-22T15:35:39.394 INFO:journalctl@ceph.mon.a.smithi145.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-22T15:35:39.394 INFO:journalctl@ceph.mon.a.smithi145.stdout: 2024-04-22T15:35:39.394 INFO:journalctl@ceph.mon.a.smithi145.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T15:35:39.395 INFO:journalctl@ceph.mon.a.smithi145.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-22T15:35:39.395 INFO:journalctl@ceph.mon.a.smithi145.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-22T15:35:39.395 INFO:journalctl@ceph.mon.a.smithi145.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-22T15:35:39.395 INFO:journalctl@ceph.mon.a.smithi145.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-22T15:35:39.395 INFO:journalctl@ceph.mon.a.smithi145.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-22T15:35:39.395 INFO:journalctl@ceph.mon.a.smithi145.stdout: Cumulative compaction: 0.00 GB write, 58.89 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T15:35:39.395 INFO:journalctl@ceph.mon.a.smithi145.stdout: Interval compaction: 0.00 GB write, 58.89 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T15:35:39.395 INFO:journalctl@ceph.mon.a.smithi145.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-22T15:35:39.395 INFO:journalctl@ceph.mon.a.smithi145.stdout: Block cache BinnedLRUCache@0x56209d3291f0#2 capacity: 512.00 MB usage: 1.45 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2024-04-22T15:35:39.395 INFO:journalctl@ceph.mon.a.smithi145.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.50 KB,9.53674e-05%) IndexBlock(1,0.95 KB,0.000181794%) Misc(1,0.00 KB,0%) 2024-04-22T15:35:39.395 INFO:journalctl@ceph.mon.a.smithi145.stdout: 2024-04-22T15:35:39.395 INFO:journalctl@ceph.mon.a.smithi145.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-22T15:35:39.395 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: starting mon.a rank 0 at public addrs [v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0] at bind addrs [v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:35:39.395 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: mon.a@-1(???) e3 preinit fsid 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:35:39.395 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: mon.a@-1(???).mds e1 new map 2024-04-22T15:35:39.396 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: mon.a@-1(???).mds e1 print_map 2024-04-22T15:35:39.396 INFO:journalctl@ceph.mon.a.smithi145.stdout: e1 2024-04-22T15:35:39.396 INFO:journalctl@ceph.mon.a.smithi145.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-22T15:35:39.396 INFO:journalctl@ceph.mon.a.smithi145.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-22T15:35:39.396 INFO:journalctl@ceph.mon.a.smithi145.stdout: legacy client fscid: -1 2024-04-22T15:35:39.396 INFO:journalctl@ceph.mon.a.smithi145.stdout: 2024-04-22T15:35:39.396 INFO:journalctl@ceph.mon.a.smithi145.stdout: No filesystems configured 2024-04-22T15:35:39.396 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: mon.a@-1(???).osd e107 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-22T15:35:39.396 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: mon.a@-1(???).osd e107 crush map has features 432629239337189376, adjusting msgr requires 2024-04-22T15:35:39.396 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: mon.a@-1(???).osd e107 crush map has features 432629239337189376, adjusting msgr requires 2024-04-22T15:35:39.396 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: mon.a@-1(???).osd e107 crush map has features 432629239337189376, adjusting msgr requires 2024-04-22T15:35:39.396 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 ceph-mon[82523]: mon.a@-1(???).paxosservice(auth 1..26) refresh upgraded, format 0 -> 3 2024-04-22T15:35:39.396 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 bash[82505]: 7bfdc11120f4affc98db5c05200905b159038265bb472095719837905fe1fb72 2024-04-22T15:35:39.396 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:39 smithi145 systemd[1]: Started Ceph mon.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:35:42.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ignoring --setuser ceph since I am not root 2024-04-22T15:35:42.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ignoring --setgroup ceph since I am not root 2024-04-22T15:35:42.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mgr[60909]: -- 172.21.15.145:0/3084945603 <== mon.0 v2:172.21.15.145:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5574465545a0 con 0x557446528c00 2024-04-22T15:35:42.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:35:42.452+0000 7f182b025200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T15:35:42.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[82523]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:35:42.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[82523]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:35:42.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[82523]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:35:42.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:35:42.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[82523]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:35:42.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[82523]: mon.a calling monitor election 2024-04-22T15:35:42.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[82523]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:35:42.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[82523]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-22T15:35:42.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[82523]: monmap epoch 3 2024-04-22T15:35:42.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[82523]: fsid 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:35:42.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[82523]: last_changed 2024-04-22T15:06:01.062345+0000 2024-04-22T15:35:42.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[82523]: created 2024-04-22T15:04:30.924274+0000 2024-04-22T15:35:42.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[82523]: min_mon_release 17 (quincy) 2024-04-22T15:35:42.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[82523]: election_strategy: 1 2024-04-22T15:35:42.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[82523]: 0: [v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0] mon.a 2024-04-22T15:35:42.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[82523]: 1: [v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0] mon.c 2024-04-22T15:35:42.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[82523]: 2: [v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0] mon.b 2024-04-22T15:35:42.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[82523]: fsmap 2024-04-22T15:35:42.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[82523]: osdmap e107: 8 total, 8 up, 8 in 2024-04-22T15:35:42.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[82523]: mgrmap e43: y(active, since 4m), standbys: x 2024-04-22T15:35:42.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[82523]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:35:42.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[82523]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:35:42.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[82523]: daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state 2024-04-22T15:35:42.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[82523]: from='mgr.25306 ' entity='' 2024-04-22T15:35:42.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[82523]: mgrmap e44: y(active, since 4m), standbys: x 2024-04-22T15:35:42.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:35:42.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:35:42.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[26606]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:35:42.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:35:42.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[26606]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:35:42.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[26606]: mon.a calling monitor election 2024-04-22T15:35:42.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[26606]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:35:42.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[26606]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-22T15:35:42.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[26606]: monmap epoch 3 2024-04-22T15:35:42.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[26606]: fsid 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:35:42.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[26606]: last_changed 2024-04-22T15:06:01.062345+0000 2024-04-22T15:35:42.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[26606]: created 2024-04-22T15:04:30.924274+0000 2024-04-22T15:35:42.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[26606]: min_mon_release 17 (quincy) 2024-04-22T15:35:42.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[26606]: election_strategy: 1 2024-04-22T15:35:42.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[26606]: 0: [v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0] mon.a 2024-04-22T15:35:42.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[26606]: 1: [v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0] mon.c 2024-04-22T15:35:42.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[26606]: 2: [v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0] mon.b 2024-04-22T15:35:42.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[26606]: fsmap 2024-04-22T15:35:42.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[26606]: osdmap e107: 8 total, 8 up, 8 in 2024-04-22T15:35:42.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[26606]: mgrmap e43: y(active, since 4m), standbys: x 2024-04-22T15:35:42.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[26606]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:35:42.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[26606]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:35:42.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[26606]: daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state 2024-04-22T15:35:42.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[26606]: from='mgr.25306 ' entity='' 2024-04-22T15:35:42.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-mon[26606]: mgrmap e44: y(active, since 4m), standbys: x 2024-04-22T15:35:42.633 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:42 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:35:42.633 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:42 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:35:42.633 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:42 smithi172 ceph-mon[65307]: from='mgr.25306 172.21.15.145:0/1386205952' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:35:42.634 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:42 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:35:42.634 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:42 smithi172 ceph-mon[65307]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:35:42.634 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:42 smithi172 ceph-mon[65307]: mon.a calling monitor election 2024-04-22T15:35:42.634 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:42 smithi172 ceph-mon[65307]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:35:42.634 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:42 smithi172 ceph-mon[65307]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-22T15:35:42.634 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:42 smithi172 ceph-mon[65307]: monmap epoch 3 2024-04-22T15:35:42.634 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:42 smithi172 ceph-mon[65307]: fsid 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:35:42.634 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:42 smithi172 ceph-mon[65307]: last_changed 2024-04-22T15:06:01.062345+0000 2024-04-22T15:35:42.634 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:42 smithi172 ceph-mon[65307]: created 2024-04-22T15:04:30.924274+0000 2024-04-22T15:35:42.634 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:42 smithi172 ceph-mon[65307]: min_mon_release 17 (quincy) 2024-04-22T15:35:42.634 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:42 smithi172 ceph-mon[65307]: election_strategy: 1 2024-04-22T15:35:42.634 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:42 smithi172 ceph-mon[65307]: 0: [v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0] mon.a 2024-04-22T15:35:42.634 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:42 smithi172 ceph-mon[65307]: 1: [v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0] mon.c 2024-04-22T15:35:42.634 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:42 smithi172 ceph-mon[65307]: 2: [v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0] mon.b 2024-04-22T15:35:42.635 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:42 smithi172 ceph-mon[65307]: fsmap 2024-04-22T15:35:42.635 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:42 smithi172 ceph-mon[65307]: osdmap e107: 8 total, 8 up, 8 in 2024-04-22T15:35:42.635 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:42 smithi172 ceph-mon[65307]: mgrmap e43: y(active, since 4m), standbys: x 2024-04-22T15:35:42.635 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:42 smithi172 ceph-mon[65307]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:35:42.635 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:42 smithi172 ceph-mon[65307]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:35:42.635 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:42 smithi172 ceph-mon[65307]: daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state 2024-04-22T15:35:42.635 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:42 smithi172 ceph-mon[65307]: from='mgr.25306 ' entity='' 2024-04-22T15:35:42.635 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:42 smithi172 ceph-mon[65307]: mgrmap e44: y(active, since 4m), standbys: x 2024-04-22T15:35:42.635 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:42 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: ignoring --setuser ceph since I am not root 2024-04-22T15:35:42.635 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:42 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: ignoring --setgroup ceph since I am not root 2024-04-22T15:35:42.635 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:42 smithi172 ceph-mgr[58098]: -- 172.21.15.172:0/1487857666 <== mon.2 v2:172.21.15.172:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c99dd825a0 con 0x55c99dd56c00 2024-04-22T15:35:42.635 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:42 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:35:42.448+0000 7fef78285200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T15:35:42.635 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:42 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:35:42.630+0000 7fef78285200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T15:35:42.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:42 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:35:42.731+0000 7fef78285200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T15:35:43.050 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:35:42.635+0000 7f182b025200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T15:35:43.050 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:42 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:35:42.737+0000 7f182b025200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T15:35:43.050 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:35:43.046+0000 7f182b025200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T15:35:43.372 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:35:43.132+0000 7f182b025200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T15:35:43.467 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:43 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:35:43.043+0000 7fef78285200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T15:35:43.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:43 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:35:43.128+0000 7fef78285200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T15:35:44.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:44 smithi145 ceph-mon[26606]: mgrmap e45: y(active, since 4m), standbys: x 2024-04-22T15:35:44.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:44 smithi145 ceph-mon[82523]: mgrmap e45: y(active, since 4m), standbys: x 2024-04-22T15:35:44.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:44 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:35:44.340+0000 7f182b025200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T15:35:44.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:44 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:35:44.511+0000 7f182b025200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T15:35:44.658 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:44 smithi172 ceph-mon[65307]: mgrmap e45: y(active, since 4m), standbys: x 2024-04-22T15:35:44.658 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:35:44.315+0000 7fef78285200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T15:35:44.659 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:35:44.484+0000 7fef78285200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T15:35:44.980 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:44 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:35:44.684+0000 7f182b025200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T15:35:44.980 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:44 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:35:44.977+0000 7f182b025200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T15:35:45.218 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:35:44.655+0000 7fef78285200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T15:35:45.218 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:35:44.946+0000 7fef78285200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T15:35:45.218 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:45 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:35:45.040+0000 7fef78285200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T15:35:45.218 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:35:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:35:44.993Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.145:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.145:8765: connect: connection refused" 2024-04-22T15:35:45.218 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:35:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:35:44.994Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.145:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.145:8765: connect: connection refused" 2024-04-22T15:35:45.218 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:35:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:35:44.996Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.145:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.145:8765: connect: connection refused" 2024-04-22T15:35:45.218 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:35:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:35:44.996Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.145:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.145:8765: connect: connection refused" 2024-04-22T15:35:45.219 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:35:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:35:44.996Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.145:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.145:8765: connect: connection refused" 2024-04-22T15:35:45.282 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:35:45.071+0000 7f182b025200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T15:35:45.282 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:35:45.278+0000 7f182b025200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T15:35:45.622 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:35:45.475+0000 7f182b025200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T15:35:45.717 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:45 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:35:45.246+0000 7fef78285200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T15:35:45.718 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:45 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:35:45.443+0000 7fef78285200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T15:35:46.372 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:35:46.078+0000 7f182b025200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T15:35:46.467 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:46 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:35:46.046+0000 7fef78285200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T15:35:46.779 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:46 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:35:46.473+0000 7fef78285200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T15:35:46.780 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:46 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:35:46.571+0000 7fef78285200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T15:35:46.780 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:46 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:35:46.776+0000 7fef78285200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-22T15:35:46.817 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:35:46.507+0000 7f182b025200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T15:35:46.817 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:35:46.607+0000 7f182b025200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T15:35:46.818 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:35:46.814+0000 7f182b025200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-22T15:35:47.622 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:35:47.343+0000 7f182b025200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T15:35:47.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:35:47.438+0000 7f182b025200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T15:35:47.647 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:35:47.302+0000 7fef78285200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T15:35:47.648 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:35:47.396+0000 7fef78285200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T15:35:47.967 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:35:47.644+0000 7fef78285200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T15:35:47.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:35:47.778+0000 7fef78285200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T15:35:47.968 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:35:47.876+0000 7fef78285200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T15:35:48.042 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:35:47.687+0000 7f182b025200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T15:35:48.043 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:35:47.822+0000 7f182b025200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T15:35:48.043 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:35:47.920+0000 7f182b025200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T15:35:48.043 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:35:48.039+0000 7f182b025200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T15:35:48.302 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:35:47.995+0000 7fef78285200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T15:35:48.622 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:35:48.346+0000 7f182b025200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T15:35:48.717 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:35:48.299+0000 7fef78285200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T15:35:49.212 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:35:48.909+0000 7fef78285200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T15:35:49.212 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:49 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:35:49.141+0000 7fef78285200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T15:35:49.212 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:49 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: [22/Apr/2024:15:35:49] ENGINE Bus STARTING 2024-04-22T15:35:49.286 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:49 smithi145 ceph-mon[82523]: Standby manager daemon x restarted 2024-04-22T15:35:49.287 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:49 smithi145 ceph-mon[82523]: Standby manager daemon x started 2024-04-22T15:35:49.287 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:49 smithi145 ceph-mon[82523]: from='mgr.? 172.21.15.172:0/1164423434' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T15:35:49.287 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:49 smithi145 ceph-mon[82523]: from='mgr.? 172.21.15.172:0/1164423434' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:35:49.287 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:49 smithi145 ceph-mon[82523]: from='mgr.? 172.21.15.172:0/1164423434' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T15:35:49.287 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:49 smithi145 ceph-mon[82523]: from='mgr.? 172.21.15.172:0/1164423434' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:35:49.287 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:49 smithi145 ceph-mon[82523]: Active manager daemon y restarted 2024-04-22T15:35:49.287 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:49 smithi145 ceph-mon[82523]: Activating manager daemon y 2024-04-22T15:35:49.287 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:49 smithi145 ceph-mon[26606]: Standby manager daemon x restarted 2024-04-22T15:35:49.287 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:49 smithi145 ceph-mon[26606]: Standby manager daemon x started 2024-04-22T15:35:49.287 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:49 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.172:0/1164423434' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T15:35:49.287 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:49 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.172:0/1164423434' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:35:49.288 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:49 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.172:0/1164423434' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T15:35:49.288 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:49 smithi145 ceph-mon[26606]: from='mgr.? 172.21.15.172:0/1164423434' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:35:49.288 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:49 smithi145 ceph-mon[26606]: Active manager daemon y restarted 2024-04-22T15:35:49.288 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:49 smithi145 ceph-mon[26606]: Activating manager daemon y 2024-04-22T15:35:49.288 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:35:48.960+0000 7f182b025200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T15:35:49.288 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:49 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:35:49.189+0000 7f182b025200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T15:35:49.288 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:49 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:35:49] ENGINE Bus STARTING 2024-04-22T15:35:49.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:49 smithi172 ceph-mon[65307]: Standby manager daemon x restarted 2024-04-22T15:35:49.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:49 smithi172 ceph-mon[65307]: Standby manager daemon x started 2024-04-22T15:35:49.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:49 smithi172 ceph-mon[65307]: from='mgr.? 172.21.15.172:0/1164423434' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T15:35:49.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:49 smithi172 ceph-mon[65307]: from='mgr.? 172.21.15.172:0/1164423434' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T15:35:49.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:49 smithi172 ceph-mon[65307]: from='mgr.? 172.21.15.172:0/1164423434' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T15:35:49.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:49 smithi172 ceph-mon[65307]: from='mgr.? 172.21.15.172:0/1164423434' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T15:35:49.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:49 smithi172 ceph-mon[65307]: Active manager daemon y restarted 2024-04-22T15:35:49.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:49 smithi172 ceph-mon[65307]: Activating manager daemon y 2024-04-22T15:35:49.468 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:49 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: CherryPy Checker: 2024-04-22T15:35:49.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:49 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: The Application mounted at '' has an empty config. 2024-04-22T15:35:49.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:49 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: 2024-04-22T15:35:49.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:49 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: [22/Apr/2024:15:35:49] ENGINE Serving on http://:::9283 2024-04-22T15:35:49.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:35:49 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x[58094]: [22/Apr/2024:15:35:49] ENGINE Bus STARTED 2024-04-22T15:35:49.622 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:49 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CherryPy Checker: 2024-04-22T15:35:49.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:49 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: The Application mounted at '' has an empty config. 2024-04-22T15:35:49.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:49 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:35:49.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:49 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:35:49] ENGINE Serving on http://:::9283 2024-04-22T15:35:49.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:49 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:35:49] ENGINE Bus STARTED 2024-04-22T15:35:50.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:50 smithi172 ceph-mon[65307]: osdmap e108: 8 total, 8 up, 8 in 2024-04-22T15:35:50.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:50 smithi172 ceph-mon[65307]: mgrmap e46: y(active, starting, since 0.0105086s), standbys: x 2024-04-22T15:35:50.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:50 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:35:50.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:50 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:35:50.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:50 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:35:50.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:50 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T15:35:50.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:50 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T15:35:50.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:50 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:35:50.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:50 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:35:50.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:50 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:35:50.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:50 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:35:50.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:50 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:35:50.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:50 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:35:50.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:50 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:35:50.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:50 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:35:50.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:50 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:35:50.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:50 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:35:50.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:50 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:35:50.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:50 smithi172 ceph-mon[65307]: Manager daemon y is now available 2024-04-22T15:35:50.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:50 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:35:50.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:50 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:35:50.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:50 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:35:50.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:50 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:35:50.524 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[26606]: osdmap e108: 8 total, 8 up, 8 in 2024-04-22T15:35:50.525 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[26606]: mgrmap e46: y(active, starting, since 0.0105086s), standbys: x 2024-04-22T15:35:50.525 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:35:50.525 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:35:50.525 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:35:50.525 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T15:35:50.525 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T15:35:50.525 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:35:50.525 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:35:50.525 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:35:50.525 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:35:50.525 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:35:50.525 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:35:50.526 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:35:50.526 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:35:50.526 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:35:50.526 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:35:50.526 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:35:50.526 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[26606]: Manager daemon y is now available 2024-04-22T15:35:50.526 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:35:50.526 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:35:50.526 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:35:50.526 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:35:50.527 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[82523]: osdmap e108: 8 total, 8 up, 8 in 2024-04-22T15:35:50.527 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[82523]: mgrmap e46: y(active, starting, since 0.0105086s), standbys: x 2024-04-22T15:35:50.527 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:35:50.527 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:35:50.527 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:35:50.527 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T15:35:50.527 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T15:35:50.527 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:35:50.527 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:35:50.527 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:35:50.527 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:35:50.527 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:35:50.528 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:35:50.528 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:35:50.528 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:35:50.528 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T15:35:50.528 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T15:35:50.528 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T15:35:50.528 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[82523]: Manager daemon y is now available 2024-04-22T15:35:50.529 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:35:50.529 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:35:50.529 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T15:35:50.529 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T15:35:50.529 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:50 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:35:50.226+0000 7f17e84c2700 -1 mgr.server handle_report got status from non-daemon mon.a 2024-04-22T15:35:51.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:51 smithi172 ceph-mon[65307]: [22/Apr/2024:15:35:50] ENGINE Bus STARTING 2024-04-22T15:35:51.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:51 smithi172 ceph-mon[65307]: mgrmap e47: y(active, since 1.03056s), standbys: x 2024-04-22T15:35:51.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:51 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:35:51.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:51 smithi172 ceph-mon[65307]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:35:51.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:51 smithi172 ceph-mon[65307]: [22/Apr/2024:15:35:50] ENGINE Serving on http://172.21.15.145:8765 2024-04-22T15:35:51.484 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:51 smithi145 ceph-mon[26606]: [22/Apr/2024:15:35:50] ENGINE Bus STARTING 2024-04-22T15:35:51.484 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:51 smithi145 ceph-mon[26606]: mgrmap e47: y(active, since 1.03056s), standbys: x 2024-04-22T15:35:51.484 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:51 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:35:51.484 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:51 smithi145 ceph-mon[26606]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:35:51.484 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:51 smithi145 ceph-mon[26606]: [22/Apr/2024:15:35:50] ENGINE Serving on http://172.21.15.145:8765 2024-04-22T15:35:51.485 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:51 smithi145 ceph-mon[82523]: [22/Apr/2024:15:35:50] ENGINE Bus STARTING 2024-04-22T15:35:51.485 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:51 smithi145 ceph-mon[82523]: mgrmap e47: y(active, since 1.03056s), standbys: x 2024-04-22T15:35:51.485 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:51 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:35:51.485 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:51 smithi145 ceph-mon[82523]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:35:51.485 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:51 smithi145 ceph-mon[82523]: [22/Apr/2024:15:35:50] ENGINE Serving on http://172.21.15.145:8765 2024-04-22T15:35:52.370 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:52 smithi145 ceph-mon[26606]: [22/Apr/2024:15:35:50] ENGINE Serving on https://172.21.15.145:7150 2024-04-22T15:35:52.370 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:52 smithi145 ceph-mon[26606]: [22/Apr/2024:15:35:50] ENGINE Bus STARTED 2024-04-22T15:35:52.371 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:52 smithi145 ceph-mon[26606]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:35:52.371 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:52 smithi145 ceph-mon[26606]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state']) 2024-04-22T15:35:52.371 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:52 smithi145 ceph-mon[26606]: Cluster is now healthy 2024-04-22T15:35:52.371 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:52 smithi145 ceph-mon[26606]: mgrmap e48: y(active, since 2s), standbys: x 2024-04-22T15:35:52.371 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:52 smithi145 ceph-mon[82523]: [22/Apr/2024:15:35:50] ENGINE Serving on https://172.21.15.145:7150 2024-04-22T15:35:52.371 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:52 smithi145 ceph-mon[82523]: [22/Apr/2024:15:35:50] ENGINE Bus STARTED 2024-04-22T15:35:52.371 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:52 smithi145 ceph-mon[82523]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:35:52.371 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:52 smithi145 ceph-mon[82523]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state']) 2024-04-22T15:35:52.371 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:52 smithi145 ceph-mon[82523]: Cluster is now healthy 2024-04-22T15:35:52.371 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:52 smithi145 ceph-mon[82523]: mgrmap e48: y(active, since 2s), standbys: x 2024-04-22T15:35:52.569 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:52 smithi172 ceph-mon[65307]: [22/Apr/2024:15:35:50] ENGINE Serving on https://172.21.15.145:7150 2024-04-22T15:35:52.569 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:52 smithi172 ceph-mon[65307]: [22/Apr/2024:15:35:50] ENGINE Bus STARTED 2024-04-22T15:35:52.569 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:52 smithi172 ceph-mon[65307]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:35:52.569 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:52 smithi172 ceph-mon[65307]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state']) 2024-04-22T15:35:52.569 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:52 smithi172 ceph-mon[65307]: Cluster is now healthy 2024-04-22T15:35:52.569 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:52 smithi172 ceph-mon[65307]: mgrmap e48: y(active, since 2s), standbys: x 2024-04-22T15:35:54.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:54 smithi145 ceph-mon[26606]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:35:54.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:54 smithi145 ceph-mon[26606]: mgrmap e49: y(active, since 4s), standbys: x 2024-04-22T15:35:54.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:54 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:35:54.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:54 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:35:54.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:54 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:35:54.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:54 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:35:54.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:54 smithi145 ceph-mon[82523]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:35:54.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:54 smithi145 ceph-mon[82523]: mgrmap e49: y(active, since 4s), standbys: x 2024-04-22T15:35:54.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:54 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:35:54.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:54 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:35:54.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:54 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:35:54.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:54 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:35:54.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:54 smithi172 ceph-mon[65307]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:35:54.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:54 smithi172 ceph-mon[65307]: mgrmap e49: y(active, since 4s), standbys: x 2024-04-22T15:35:54.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:54 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:35:54.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:54 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:35:54.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:54 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:35:54.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:54 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:35:55.622 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:35:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:35:55] "GET /metrics HTTP/1.1" 200 35219 "" "Prometheus/2.43.0" 2024-04-22T15:35:56.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:55 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:35:56.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:55 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:35:56.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:55 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:35:56.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:55 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:35:56.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:55 smithi145 ceph-mon[82523]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:35:56.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:55 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:35:56.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:55 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:35:56.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:55 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:35:56.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:55 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:35:56.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:55 smithi145 ceph-mon[26606]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:35:56.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:55 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:35:56.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:55 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:35:56.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:55 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:35:56.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:55 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:35:56.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:55 smithi172 ceph-mon[65307]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:35:58.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:58 smithi172 ceph-mon[65307]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-22T15:35:58.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:58 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:35:58.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:58 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:35:58.612 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:58 smithi145 ceph-mon[82523]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-22T15:35:58.612 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:58 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:35:58.612 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:58 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:35:58.612 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:58 smithi145 ceph-mon[26606]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-22T15:35:58.612 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:58 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:35:58.612 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:58 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:35:59.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:59 smithi172 ceph-mon[65307]: Detected new or changed devices on smithi145 2024-04-22T15:35:59.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:59 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:35:59.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:59 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:35:59.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:59 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:35:59.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:59 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:35:59.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:59 smithi172 ceph-mon[65307]: Updating smithi145:/etc/ceph/ceph.conf 2024-04-22T15:35:59.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:59 smithi172 ceph-mon[65307]: Updating smithi172:/etc/ceph/ceph.conf 2024-04-22T15:35:59.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:59 smithi172 ceph-mon[65307]: Updating smithi172:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:35:59.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:59 smithi172 ceph-mon[65307]: Updating smithi145:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:35:59.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:59 smithi172 ceph-mon[65307]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-22T15:35:59.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:59 smithi172 ceph-mon[65307]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:35:59.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:35:59 smithi172 ceph-mon[65307]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:36:00.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:59 smithi145 ceph-mon[82523]: Detected new or changed devices on smithi145 2024-04-22T15:36:00.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:59 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:00.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:59 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:00.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:59 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:00.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:59 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:36:00.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:59 smithi145 ceph-mon[82523]: Updating smithi145:/etc/ceph/ceph.conf 2024-04-22T15:36:00.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:59 smithi145 ceph-mon[82523]: Updating smithi172:/etc/ceph/ceph.conf 2024-04-22T15:36:00.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:59 smithi145 ceph-mon[82523]: Updating smithi172:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:36:00.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:59 smithi145 ceph-mon[82523]: Updating smithi145:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:36:00.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:59 smithi145 ceph-mon[82523]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-22T15:36:00.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:59 smithi145 ceph-mon[82523]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:36:00.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:35:59 smithi145 ceph-mon[82523]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:36:00.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:59 smithi145 ceph-mon[26606]: Detected new or changed devices on smithi145 2024-04-22T15:36:00.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:59 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:00.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:59 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:00.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:59 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:00.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:59 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:36:00.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:59 smithi145 ceph-mon[26606]: Updating smithi145:/etc/ceph/ceph.conf 2024-04-22T15:36:00.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:59 smithi145 ceph-mon[26606]: Updating smithi172:/etc/ceph/ceph.conf 2024-04-22T15:36:00.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:59 smithi145 ceph-mon[26606]: Updating smithi172:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:36:00.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:59 smithi145 ceph-mon[26606]: Updating smithi145:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-22T15:36:00.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:59 smithi145 ceph-mon[26606]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-22T15:36:00.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:59 smithi145 ceph-mon[26606]: Updating smithi172:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:36:00.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:35:59 smithi145 ceph-mon[26606]: Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-04-22T15:36:00.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:00 smithi145 ceph-mon[26606]: Updating smithi172:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:36:00.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:00 smithi145 ceph-mon[26606]: Updating smithi145:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:36:00.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:00 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:00.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:00 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:00.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:00 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:00.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:00 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:00.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:00 smithi145 ceph-mon[26606]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-22T15:36:00.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:00 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:00.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:00 smithi145 ceph-mon[26606]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:36:00.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:00 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:36:00.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:00 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:00.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:00 smithi145 ceph-mon[26606]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:36:00.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:00 smithi145 ceph-mon[26606]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:36:00.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:00 smithi145 ceph-mon[82523]: Updating smithi172:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:36:00.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:00 smithi145 ceph-mon[82523]: Updating smithi145:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:36:00.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:00 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:00.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:00 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:00.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:00 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:00.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:00 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:00.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:00 smithi145 ceph-mon[82523]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-22T15:36:00.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:00 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:00.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:00 smithi145 ceph-mon[82523]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:36:00.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:00 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:36:00.875 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:00 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:00.875 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:00 smithi145 ceph-mon[82523]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:36:00.875 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:00 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:36:01.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:00 smithi172 ceph-mon[65307]: Updating smithi172:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:36:01.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:00 smithi172 ceph-mon[65307]: Updating smithi145:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-22T15:36:01.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:00 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:01.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:00 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:01.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:00 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:01.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:00 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:01.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:00 smithi172 ceph-mon[65307]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-22T15:36:01.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:00 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:01.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:00 smithi172 ceph-mon[65307]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:36:01.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:00 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:36:01.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:00 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:01.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:00 smithi172 ceph-mon[65307]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:36:01.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:00 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:36:01.751 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:36:01.609+0000 7f17b18ae700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:36:01.751 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:36:01.751 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:36:01.751 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:36:01.751 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:36:01.751 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:01.752 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:01.752 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:36:01.752 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:36:01.752 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:36:01.752 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:36:01.752 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:36:01.752 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:36:01.752 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:36:01.752 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:36:01.752 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:36:01.752 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:36:01.752 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:36:01.752 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:36:01.753 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:01.753 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:01.753 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:36:01.753 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:36:01.753 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:36:01.753 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:36:01.753 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:36:01.754 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:36:01.754 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:36:01.754 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:36:01.754 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:36:01.754 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:36:01.754 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:36:01.754 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:36:01.754 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:36:01.754 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:36:01.754 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:36:01.754 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:36:01.754 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:36:01.754 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:36:01.755 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:36:01.755 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:36:01.755 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:36:01.755 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:36:01.755 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:01.755 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:01.755 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:36:01.755 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:36:01.755 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:36:01.755 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:36:01.755 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:36:01.755 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:36:01.755 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:36:01.755 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:36:01.755 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:36:01.755 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:36:01.756 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:36:01.756 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:36:01.756 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:01.756 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:02.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-mon[82523]: Health check failed: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] (CEPHADM_FAILED_DAEMON) 2024-04-22T15:36:02.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:36:02.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:02.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-22T15:36:02.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-22T15:36:02.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-mon[26606]: Health check failed: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] (CEPHADM_FAILED_DAEMON) 2024-04-22T15:36:02.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:36:02.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:02.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-22T15:36:02.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:01 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-22T15:36:02.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:01 smithi172 ceph-mon[65307]: Health check failed: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] (CEPHADM_FAILED_DAEMON) 2024-04-22T15:36:02.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:01 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:36:02.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:01 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:02.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:01 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-22T15:36:02.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:01 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-22T15:36:02.846 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:02 smithi145 ceph-mon[26606]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:36:02.846 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:36:02.846 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:36:02.846 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:36:02.846 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:36:02.846 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:02.846 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:02.846 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:36:02.846 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:36:02.847 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:36:02.847 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:36:02.847 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:36:02.847 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:36:02.847 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:36:02.847 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:36:02.847 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:36:02.847 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:36:02.847 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:36:02.847 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:36:02.847 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:02.847 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:02.847 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:36:02.847 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:36:02.847 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:36:02.848 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:36:02.848 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:36:02.848 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:36:02.849 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:36:02.849 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:36:02.849 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:36:02.849 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:36:02.849 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:36:02.849 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:36:02.850 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:36:02.850 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:36:02.850 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:36:02.850 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:36:02.850 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:36:02.850 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:36:02.850 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:36:02.850 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:36:02.850 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:36:02.850 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:36:02.850 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:02.850 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:02.850 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:36:02.850 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:36:02.850 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:36:02.851 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:36:02.851 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:36:02.851 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:36:02.851 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:36:02.851 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:36:02.851 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:36:02.851 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:02 smithi145 ceph-mon[82523]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:36:02.851 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:36:02.851 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:36:02.851 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:36:02.851 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:36:02.852 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:02.852 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:02.852 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:36:02.852 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:36:02.852 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:36:02.852 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:36:02.852 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:36:02.852 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:36:02.852 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:36:02.852 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:36:02.852 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:36:02.852 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:36:02.852 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:36:02.852 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:36:02.853 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:02.853 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:02.853 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:36:02.853 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:36:02.853 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:36:02.853 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:36:02.853 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:36:02.853 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:36:02.853 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:36:02.853 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:36:02.853 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:36:02.853 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:36:02.853 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:36:02.853 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:36:02.853 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:36:02.853 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:36:02.854 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:36:02.854 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:36:02.854 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:36:02.854 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:36:02.854 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:36:02.854 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:36:02.854 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:36:02.854 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:36:02.854 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:02.854 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:02.854 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:36:02.854 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:36:02.854 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:36:02.854 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:36:02.854 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:36:02.855 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:36:02.855 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:36:02.855 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:36:02.855 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:36:02.855 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:36:02.855 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:36:02.855 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:36:02.855 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:02.855 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:02.855 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:02 smithi145 ceph-mon[82523]: Upgrade: It appears safe to stop mon.c 2024-04-22T15:36:02.855 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:02 smithi145 ceph-mon[82523]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T15:36:02.855 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:36:02.856 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:36:02.856 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:36:02.856 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:02.856 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:02.856 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:02 smithi145 ceph-mon[26606]: Upgrade: It appears safe to stop mon.c 2024-04-22T15:36:02.856 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:02 smithi145 ceph-mon[26606]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T15:36:03.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:02 smithi172 ceph-mon[65307]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:36:03.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:36:03.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:36:03.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:36:03.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:36:03.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:03.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:03.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:36:03.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:36:03.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:36:03.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:36:03.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:36:03.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:36:03.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:36:03.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:36:03.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:36:03.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:36:03.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:36:03.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:36:03.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:03.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:03.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:36:03.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:36:03.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:36:03.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:36:03.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:36:03.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:36:03.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:36:03.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:36:03.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:36:03.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:36:03.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:36:03.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:36:03.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:36:03.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:36:03.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:36:03.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:36:03.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:36:03.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:36:03.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:36:03.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:36:03.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:36:03.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:36:03.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:03.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:03.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:36:03.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:36:03.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:36:03.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:36:03.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:36:03.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:36:03.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:36:03.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:36:03.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:36:03.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:36:03.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:36:03.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:36:03.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:03.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:03.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:02 smithi172 ceph-mon[65307]: Upgrade: It appears safe to stop mon.c 2024-04-22T15:36:03.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:02 smithi172 ceph-mon[65307]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T15:36:04.353 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:36:04.353 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:04 smithi145 ceph-mon[82523]: Upgrade: Updating mon.c 2024-04-22T15:36:04.353 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:04 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:04.353 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:04 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:36:04.353 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:04 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:36:04.353 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:04 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:04.353 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:04 smithi145 ceph-mon[82523]: Deploying daemon mon.c on smithi145 2024-04-22T15:36:04.354 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:04 smithi145 ceph-mon[26606]: Upgrade: Updating mon.c 2024-04-22T15:36:04.354 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:04 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:04.354 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:04 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:36:04.354 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:04 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:36:04.354 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:04 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:04.354 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:04 smithi145 ceph-mon[26606]: Deploying daemon mon.c on smithi145 2024-04-22T15:36:04.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:04 smithi172 ceph-mon[65307]: Upgrade: Updating mon.c 2024-04-22T15:36:04.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:04 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:04.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:04 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:36:04.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:04 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:36:04.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:04 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:04.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:04 smithi172 ceph-mon[65307]: Deploying daemon mon.c on smithi145 2024-04-22T15:36:05.034 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:36:05.035 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (2m) 10s ago 25m 16.6M - 0.25.0 c8568f914cd2 1c58ccf37ffd 2024-04-22T15:36:05.035 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (2m) 11s ago 24m 79.5M - 9.4.12 2bacad6d85d8 2f825d15ab10 2024-04-22T15:36:05.035 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 unknown 10s ago 24m 59.8M - 3.5 e1d6a67b021e 9704631ffa0e 2024-04-22T15:36:05.035 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443,9283,8765 running (4m) 11s ago 29m 448M - 19.0.0-3223-g5883d494 3d983c26e278 e7500ef71604 2024-04-22T15:36:05.035 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (14m) 10s ago 31m 473M - 19.0.0-3223-g5883d494 3d983c26e278 b2de97566cf4 2024-04-22T15:36:05.035 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (26s) 10s ago 31m 29.8M 2048M 19.0.0-3223-g5883d494 3d983c26e278 7bfdc11120f4 2024-04-22T15:36:05.035 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (76s) 11s ago 30m 39.4M 2048M 19.0.0-3223-g5883d494 3d983c26e278 0a360b08a6d6 2024-04-22T15:36:05.035 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (30m) 10s ago 30m 60.5M 2048M 17.2.0 e1d6a67b021e b71d9b9f8509 2024-04-22T15:36:05.035 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (3m) 10s ago 25m 8426k - 1.5.0 0da6a335fe13 8c75c26fec05 2024-04-22T15:36:05.035 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (3m) 11s ago 25m 15.9M - 1.5.0 0da6a335fe13 8fd2949e2eaf 2024-04-22T15:36:05.035 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (29m) 10s ago 29m 57.1M 2992M 17.2.0 e1d6a67b021e ea9cdc639916 2024-04-22T15:36:05.035 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (29m) 10s ago 29m 55.3M 2992M 17.2.0 e1d6a67b021e 3b9b37a8781f 2024-04-22T15:36:05.035 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (28m) 10s ago 28m 50.8M 2992M 17.2.0 e1d6a67b021e d9502dded8f8 2024-04-22T15:36:05.035 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (28m) 10s ago 28m 53.1M 2992M 17.2.0 e1d6a67b021e 8c72f30e54fc 2024-04-22T15:36:05.036 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (27m) 11s ago 27m 57.8M 3248M 17.2.0 e1d6a67b021e 80001f97ed35 2024-04-22T15:36:05.036 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (27m) 11s ago 27m 54.2M 3248M 17.2.0 e1d6a67b021e ec3ab1335b5c 2024-04-22T15:36:05.036 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (27m) 11s ago 27m 54.9M 3248M 17.2.0 e1d6a67b021e 45d475edf1ca 2024-04-22T15:36:05.036 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (26m) 11s ago 26m 53.0M 3248M 17.2.0 e1d6a67b021e cc9e1d9e7916 2024-04-22T15:36:05.036 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (3m) 11s ago 26m 63.9M - 2.43.0 a07b618ecd1d 51cf4df6e8d4 2024-04-22T15:36:05.036 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (24m) 10s ago 24m 78.7M - 17.2.0 e1d6a67b021e c90a636c947e 2024-04-22T15:36:05.036 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (24m) 11s ago 24m 76.5M - 17.2.0 e1d6a67b021e 20ac46e9bfda 2024-04-22T15:36:05.290 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:05 smithi145 ceph-mon[82523]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T15:36:05.290 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:05 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:36:05.291 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:04 smithi145 systemd[1]: Stopping Ceph mon.c for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:36:05.291 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:05 smithi145 ceph-mon[26606]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T15:36:05.291 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:05 smithi145 ceph-mon[26606]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:36:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:05 smithi172 ceph-mon[65307]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-22T15:36:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:05 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:36:05.468 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:36:05.469 INFO:teuthology.orchestra.run.smithi145.stdout: "mon": { 2024-04-22T15:36:05.469 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-22T15:36:05.469 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:36:05.469 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:36:05.469 INFO:teuthology.orchestra.run.smithi145.stdout: "mgr": { 2024-04-22T15:36:05.469 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:36:05.469 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:36:05.469 INFO:teuthology.orchestra.run.smithi145.stdout: "osd": { 2024-04-22T15:36:05.469 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T15:36:05.469 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:36:05.469 INFO:teuthology.orchestra.run.smithi145.stdout: "rgw": { 2024-04-22T15:36:05.469 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:36:05.469 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:36:05.469 INFO:teuthology.orchestra.run.smithi145.stdout: "overall": { 2024-04-22T15:36:05.470 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-04-22T15:36:05.470 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 4 2024-04-22T15:36:05.470 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:36:05.470 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:36:05.622 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:05 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:36:05] "GET /metrics HTTP/1.1" 200 38019 "" "Prometheus/2.43.0" 2024-04-22T15:36:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:05 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-c[26602]: 2024-04-22T15:36:05.289+0000 7ff86ab00700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:36:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:05 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-c[26602]: 2024-04-22T15:36:05.289+0000 7ff86ab00700 -1 mon.c@1(peon) e3 *** Got Signal Terminated *** 2024-04-22T15:36:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:05 smithi145 podman[87971]: 2024-04-22 15:36:05.371461609 +0000 UTC m=+0.209600763 container died b71d9b9f850909b4f0dbe9e34be195dea8120952106575a706c786235bbb4642 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-c, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, release=754, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , ceph=True, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, vcs-type=git, io.buildah.version=1.19.8, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream) 2024-04-22T15:36:05.813 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:36:05.813 INFO:teuthology.orchestra.run.smithi145.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:36:05.813 INFO:teuthology.orchestra.run.smithi145.stdout: "in_progress": true, 2024-04-22T15:36:05.813 INFO:teuthology.orchestra.run.smithi145.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi145", 2024-04-22T15:36:05.813 INFO:teuthology.orchestra.run.smithi145.stdout: "services_complete": [], 2024-04-22T15:36:05.813 INFO:teuthology.orchestra.run.smithi145.stdout: "progress": "1/2 daemons upgraded", 2024-04-22T15:36:05.813 INFO:teuthology.orchestra.run.smithi145.stdout: "message": "Currently upgrading mon daemons", 2024-04-22T15:36:05.813 INFO:teuthology.orchestra.run.smithi145.stdout: "is_paused": false 2024-04-22T15:36:05.813 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:36:06.122 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:05 smithi145 podman[87971]: 2024-04-22 15:36:05.764700133 +0000 UTC m=+0.602839298 container cleanup b71d9b9f850909b4f0dbe9e34be195dea8120952106575a706c786235bbb4642 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-c, com.redhat.component=centos-stream-container, vcs-type=git, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, name=centos-stream, io.openshift.expose-services=, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, ceph=True, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870) 2024-04-22T15:36:06.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:05 smithi145 bash[87971]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-c 2024-04-22T15:36:06.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:05 smithi145 podman[87987]: 2024-04-22 15:36:05.941028733 +0000 UTC m=+0.568655908 container remove b71d9b9f850909b4f0dbe9e34be195dea8120952106575a706c786235bbb4642 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-c, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, RELEASE=HEAD, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, release=754, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, distribution-scope=public, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, io.k8s.display-name=CentOS Stream 8) 2024-04-22T15:36:06.593 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mon.c.service: Deactivated successfully. 2024-04-22T15:36:06.593 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 systemd[1]: Stopped Ceph mon.c for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:36:06.593 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mon.c.service: Consumed 15.520s CPU time. 2024-04-22T15:36:06.593 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 systemd[1]: Starting Ceph mon.c for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:36:06.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 podman[88143]: 2024-04-22 15:36:06.493444145 +0000 UTC m=+0.018976471 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:36:06.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 podman[88143]: 2024-04-22 15:36:06.626137317 +0000 UTC m=+0.151669649 container create 38dcfa677c181defaaf0e05ccaa8446ede97daeedabb7150cb2dec4bddda3ff2 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-c, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T15:36:07.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 podman[88143]: 2024-04-22 15:36:06.87092442 +0000 UTC m=+0.396456758 container init 38dcfa677c181defaaf0e05ccaa8446ede97daeedabb7150cb2dec4bddda3ff2 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-c, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:36:07.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 podman[88143]: 2024-04-22 15:36:06.874485061 +0000 UTC m=+0.400017388 container start 38dcfa677c181defaaf0e05ccaa8446ede97daeedabb7150cb2dec4bddda3ff2 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-c, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=) 2024-04-22T15:36:07.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: set uid:gid to 167:167 (ceph:ceph) 2024-04-22T15:36:07.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev), process ceph-mon, pid 2 2024-04-22T15:36:07.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: pidfile_write: ignore empty --pid-file 2024-04-22T15:36:07.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: load: jerasure load: lrc 2024-04-22T15:36:07.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: RocksDB version: 7.9.2 2024-04-22T15:36:07.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Git sha 0 2024-04-22T15:36:07.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Compile date 2024-04-22 12:55:29 2024-04-22T15:36:07.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: DB SUMMARY 2024-04-22T15:36:07.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: DB Session ID: SHQNZHTZWJ655D846T2Q 2024-04-22T15:36:07.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: CURRENT file: CURRENT 2024-04-22T15:36:07.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: IDENTITY file: IDENTITY 2024-04-22T15:36:07.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: MANIFEST file: MANIFEST-000009 size: 3195 Bytes 2024-04-22T15:36:07.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000060.sst 2024-04-22T15:36:07.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000058.log size: 2538689 ; 2024-04-22T15:36:07.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.error_if_exists: 0 2024-04-22T15:36:07.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.create_if_missing: 0 2024-04-22T15:36:07.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.paranoid_checks: 1 2024-04-22T15:36:07.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-22T15:36:07.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-22T15:36:07.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-22T15:36:07.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.env: 0x5599d1cc4cc0 2024-04-22T15:36:07.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.fs: PosixFileSystem 2024-04-22T15:36:07.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.info_log: 0x5599d3aedbe0 2024-04-22T15:36:07.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.max_file_opening_threads: 16 2024-04-22T15:36:07.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.statistics: (nil) 2024-04-22T15:36:07.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.use_fsync: 0 2024-04-22T15:36:07.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.max_log_file_size: 0 2024-04-22T15:36:07.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-22T15:36:07.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-22T15:36:07.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.keep_log_file_num: 1000 2024-04-22T15:36:07.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.recycle_log_file_num: 0 2024-04-22T15:36:07.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.allow_fallocate: 1 2024-04-22T15:36:07.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.allow_mmap_reads: 0 2024-04-22T15:36:07.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.allow_mmap_writes: 0 2024-04-22T15:36:07.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.use_direct_reads: 0 2024-04-22T15:36:07.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-22T15:36:07.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.create_missing_column_families: 0 2024-04-22T15:36:07.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.db_log_dir: 2024-04-22T15:36:07.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.wal_dir: 2024-04-22T15:36:07.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-22T15:36:07.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-22T15:36:07.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-22T15:36:07.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-22T15:36:07.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-22T15:36:07.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-22T15:36:07.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.advise_random_on_open: 1 2024-04-22T15:36:07.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.db_write_buffer_size: 0 2024-04-22T15:36:07.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.write_buffer_manager: 0x5599d3a81220 2024-04-22T15:36:07.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-22T15:36:07.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-22T15:36:07.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-22T15:36:07.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.rate_limiter: (nil) 2024-04-22T15:36:07.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-22T15:36:07.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.wal_recovery_mode: 2 2024-04-22T15:36:07.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.enable_thread_tracking: 0 2024-04-22T15:36:07.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.enable_pipelined_write: 0 2024-04-22T15:36:07.378 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.unordered_write: 0 2024-04-22T15:36:07.378 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-22T15:36:07.378 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-22T15:36:07.378 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-22T15:36:07.378 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-22T15:36:07.378 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.row_cache: None 2024-04-22T15:36:07.378 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.wal_filter: None 2024-04-22T15:36:07.379 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-22T15:36:07.379 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.allow_ingest_behind: 0 2024-04-22T15:36:07.379 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.two_write_queues: 0 2024-04-22T15:36:07.379 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.manual_wal_flush: 0 2024-04-22T15:36:07.379 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.wal_compression: 0 2024-04-22T15:36:07.379 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.atomic_flush: 0 2024-04-22T15:36:07.380 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-22T15:36:07.380 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-22T15:36:07.380 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-22T15:36:07.380 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.log_readahead_size: 0 2024-04-22T15:36:07.380 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-22T15:36:07.380 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.best_efforts_recovery: 0 2024-04-22T15:36:07.380 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-22T15:36:07.380 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-22T15:36:07.380 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.allow_data_in_errors: 0 2024-04-22T15:36:07.380 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.db_host_id: __hostname__ 2024-04-22T15:36:07.380 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-22T15:36:07.380 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.max_background_jobs: 2 2024-04-22T15:36:07.380 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.max_background_compactions: -1 2024-04-22T15:36:07.380 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.max_subcompactions: 1 2024-04-22T15:36:07.380 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-22T15:36:07.380 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-22T15:36:07.381 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-22T15:36:07.381 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.max_total_wal_size: 0 2024-04-22T15:36:07.381 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-22T15:36:07.381 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-22T15:36:07.381 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-22T15:36:07.381 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-22T15:36:07.381 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.max_open_files: -1 2024-04-22T15:36:07.381 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.bytes_per_sync: 0 2024-04-22T15:36:07.381 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-22T15:36:07.381 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-22T15:36:07.381 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.compaction_readahead_size: 0 2024-04-22T15:36:07.381 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.max_background_flushes: -1 2024-04-22T15:36:07.381 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Compression algorithms supported: 2024-04-22T15:36:07.381 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-22T15:36:07.381 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: kZSTD supported: 0 2024-04-22T15:36:07.382 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: kXpressCompression supported: 0 2024-04-22T15:36:07.382 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: kLZ4HCCompression supported: 1 2024-04-22T15:36:07.382 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: kZlibCompression supported: 1 2024-04-22T15:36:07.382 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: kSnappyCompression supported: 1 2024-04-22T15:36:07.382 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: kLZ4Compression supported: 1 2024-04-22T15:36:07.382 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: kBZip2Compression supported: 0 2024-04-22T15:36:07.382 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-22T15:36:07.382 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-22T15:36:07.382 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-04-22T15:36:07.382 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-22T15:36:07.382 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-22T15:36:07.382 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.merge_operator: 2024-04-22T15:36:07.382 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.compaction_filter: None 2024-04-22T15:36:07.382 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.compaction_filter_factory: None 2024-04-22T15:36:07.382 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.sst_partitioner_factory: None 2024-04-22T15:36:07.383 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-22T15:36:07.383 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-22T15:36:07.383 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5599d3aeddc0) 2024-04-22T15:36:07.383 INFO:journalctl@ceph.mon.c.smithi145.stdout: cache_index_and_filter_blocks: 1 2024-04-22T15:36:07.383 INFO:journalctl@ceph.mon.c.smithi145.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-22T15:36:07.383 INFO:journalctl@ceph.mon.c.smithi145.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-22T15:36:07.383 INFO:journalctl@ceph.mon.c.smithi145.stdout: pin_top_level_index_and_filter: 1 2024-04-22T15:36:07.383 INFO:journalctl@ceph.mon.c.smithi145.stdout: index_type: 0 2024-04-22T15:36:07.383 INFO:journalctl@ceph.mon.c.smithi145.stdout: data_block_index_type: 0 2024-04-22T15:36:07.383 INFO:journalctl@ceph.mon.c.smithi145.stdout: index_shortening: 1 2024-04-22T15:36:07.383 INFO:journalctl@ceph.mon.c.smithi145.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-22T15:36:07.383 INFO:journalctl@ceph.mon.c.smithi145.stdout: checksum: 4 2024-04-22T15:36:07.383 INFO:journalctl@ceph.mon.c.smithi145.stdout: no_block_cache: 0 2024-04-22T15:36:07.383 INFO:journalctl@ceph.mon.c.smithi145.stdout: block_cache: 0x5599d3ab31f0 2024-04-22T15:36:07.383 INFO:journalctl@ceph.mon.c.smithi145.stdout: block_cache_name: BinnedLRUCache 2024-04-22T15:36:07.384 INFO:journalctl@ceph.mon.c.smithi145.stdout: block_cache_options: 2024-04-22T15:36:07.384 INFO:journalctl@ceph.mon.c.smithi145.stdout: capacity : 536870912 2024-04-22T15:36:07.384 INFO:journalctl@ceph.mon.c.smithi145.stdout: num_shard_bits : 4 2024-04-22T15:36:07.384 INFO:journalctl@ceph.mon.c.smithi145.stdout: strict_capacity_limit : 0 2024-04-22T15:36:07.384 INFO:journalctl@ceph.mon.c.smithi145.stdout: high_pri_pool_ratio: 0.000 2024-04-22T15:36:07.384 INFO:journalctl@ceph.mon.c.smithi145.stdout: block_cache_compressed: (nil) 2024-04-22T15:36:07.384 INFO:journalctl@ceph.mon.c.smithi145.stdout: persistent_cache: (nil) 2024-04-22T15:36:07.384 INFO:journalctl@ceph.mon.c.smithi145.stdout: block_size: 4096 2024-04-22T15:36:07.384 INFO:journalctl@ceph.mon.c.smithi145.stdout: block_size_deviation: 10 2024-04-22T15:36:07.384 INFO:journalctl@ceph.mon.c.smithi145.stdout: block_restart_interval: 16 2024-04-22T15:36:07.384 INFO:journalctl@ceph.mon.c.smithi145.stdout: index_block_restart_interval: 1 2024-04-22T15:36:07.384 INFO:journalctl@ceph.mon.c.smithi145.stdout: metadata_block_size: 4096 2024-04-22T15:36:07.384 INFO:journalctl@ceph.mon.c.smithi145.stdout: partition_filters: 0 2024-04-22T15:36:07.384 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_delta_encoding: 1 2024-04-22T15:36:07.384 INFO:journalctl@ceph.mon.c.smithi145.stdout: filter_policy: bloomfilter 2024-04-22T15:36:07.385 INFO:journalctl@ceph.mon.c.smithi145.stdout: whole_key_filtering: 1 2024-04-22T15:36:07.385 INFO:journalctl@ceph.mon.c.smithi145.stdout: verify_compression: 0 2024-04-22T15:36:07.385 INFO:journalctl@ceph.mon.c.smithi145.stdout: read_amp_bytes_per_bit: 0 2024-04-22T15:36:07.385 INFO:journalctl@ceph.mon.c.smithi145.stdout: format_version: 5 2024-04-22T15:36:07.385 INFO:journalctl@ceph.mon.c.smithi145.stdout: enable_index_compression: 1 2024-04-22T15:36:07.385 INFO:journalctl@ceph.mon.c.smithi145.stdout: block_align: 0 2024-04-22T15:36:07.385 INFO:journalctl@ceph.mon.c.smithi145.stdout: max_auto_readahead_size: 262144 2024-04-22T15:36:07.385 INFO:journalctl@ceph.mon.c.smithi145.stdout: prepopulate_block_cache: 0 2024-04-22T15:36:07.385 INFO:journalctl@ceph.mon.c.smithi145.stdout: initial_auto_readahead_size: 8192 2024-04-22T15:36:07.385 INFO:journalctl@ceph.mon.c.smithi145.stdout: num_file_reads_for_auto_readahead: 2 2024-04-22T15:36:07.385 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.write_buffer_size: 33554432 2024-04-22T15:36:07.385 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.max_write_buffer_number: 2 2024-04-22T15:36:07.385 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.compression: NoCompression 2024-04-22T15:36:07.385 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.bottommost_compression: Disabled 2024-04-22T15:36:07.385 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.prefix_extractor: nullptr 2024-04-22T15:36:07.386 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-22T15:36:07.386 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.num_levels: 7 2024-04-22T15:36:07.386 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-22T15:36:07.386 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-22T15:36:07.386 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-22T15:36:07.386 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-22T15:36:07.386 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-22T15:36:07.386 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-22T15:36:07.386 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-22T15:36:07.386 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-22T15:36:07.386 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-22T15:36:07.386 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-22T15:36:07.387 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-22T15:36:07.387 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-22T15:36:07.387 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-22T15:36:07.387 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.compression_opts.level: 32767 2024-04-22T15:36:07.387 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.compression_opts.strategy: 0 2024-04-22T15:36:07.387 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-22T15:36:07.387 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-22T15:36:07.387 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-22T15:36:07.387 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-22T15:36:07.387 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.compression_opts.enabled: false 2024-04-22T15:36:07.387 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-22T15:36:07.387 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-22T15:36:07.387 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-22T15:36:07.387 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-22T15:36:07.387 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.target_file_size_base: 67108864 2024-04-22T15:36:07.388 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-22T15:36:07.388 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-22T15:36:07.388 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-22T15:36:07.388 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-22T15:36:07.388 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-22T15:36:07.388 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-22T15:36:07.388 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-22T15:36:07.388 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-22T15:36:07.388 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-22T15:36:07.388 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-22T15:36:07.388 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-22T15:36:07.388 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-22T15:36:07.388 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-22T15:36:07.388 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-22T15:36:07.388 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.arena_block_size: 1048576 2024-04-22T15:36:07.389 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-22T15:36:07.389 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-22T15:36:07.389 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.disable_auto_compactions: 0 2024-04-22T15:36:07.389 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-22T15:36:07.389 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-22T15:36:07.389 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-22T15:36:07.389 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-22T15:36:07.389 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-22T15:36:07.389 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-22T15:36:07.389 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-22T15:36:07.389 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-22T15:36:07.389 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-22T15:36:07.389 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-22T15:36:07.389 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-04-22T15:36:07.390 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.inplace_update_support: 0 2024-04-22T15:36:07.390 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-22T15:36:07.390 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-22T15:36:07.390 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-22T15:36:07.390 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-22T15:36:07.390 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.bloom_locality: 0 2024-04-22T15:36:07.390 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.max_successive_merges: 0 2024-04-22T15:36:07.390 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-22T15:36:07.390 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.paranoid_file_checks: 0 2024-04-22T15:36:07.390 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.force_consistency_checks: 1 2024-04-22T15:36:07.390 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.report_bg_io_stats: 0 2024-04-22T15:36:07.390 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.ttl: 2592000 2024-04-22T15:36:07.390 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-22T15:36:07.390 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-22T15:36:07.391 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-22T15:36:07.391 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.enable_blob_files: false 2024-04-22T15:36:07.391 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.min_blob_size: 0 2024-04-22T15:36:07.391 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.blob_file_size: 268435456 2024-04-22T15:36:07.391 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-22T15:36:07.391 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-22T15:36:07.391 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-22T15:36:07.391 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-22T15:36:07.391 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-22T15:36:07.391 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.blob_file_starting_level: 0 2024-04-22T15:36:07.391 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-22T15:36:07.391 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 60.sst 2024-04-22T15:36:07.391 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-22T15:36:07.391 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 62, last_sequence is 32650, log_number is 58,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-22T15:36:07.392 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 58 2024-04-22T15:36:07.392 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 43454046-25dc-419a-94a6-87934881bebd 2024-04-22T15:36:07.392 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713800166904590, "job": 1, "event": "recovery_started", "wal_files": [58]} 2024-04-22T15:36:07.392 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #58 mode 2 2024-04-22T15:36:07.392 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713800166910873, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 63, "file_size": 1605016, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 32647, "largest_seqno": 33093, "table_properties": {"data_size": 1601861, "index_size": 1539, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 5161, "raw_average_key_size": 24, "raw_value_size": 1597011, "raw_average_value_size": 7462, "num_data_blocks": 69, "num_entries": 214, "num_filter_entries": 214, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1713800166, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "43454046-25dc-419a-94a6-87934881bebd", "db_session_id": "SHQNZHTZWJ655D846T2Q", "orig_file_number": 63, "seqno_to_time_mapping": "N/A"}} 2024-04-22T15:36:07.392 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713800166910997, "job": 1, "event": "recovery_finished"} 2024-04-22T15:36:07.392 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: [db/version_set.cc:5047] Creating manifest 65 2024-04-22T15:36:07.392 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-22T15:36:07.392 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000058.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-22T15:36:07.392 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5599d3bc8000 2024-04-22T15:36:07.392 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: DB pointer 0x5599d3bb2000 2024-04-22T15:36:07.392 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-22T15:36:07.392 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-22T15:36:07.392 INFO:journalctl@ceph.mon.c.smithi145.stdout: ** DB Stats ** 2024-04-22T15:36:07.393 INFO:journalctl@ceph.mon.c.smithi145.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T15:36:07.393 INFO:journalctl@ceph.mon.c.smithi145.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-22T15:36:07.393 INFO:journalctl@ceph.mon.c.smithi145.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-22T15:36:07.393 INFO:journalctl@ceph.mon.c.smithi145.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T15:36:07.393 INFO:journalctl@ceph.mon.c.smithi145.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-22T15:36:07.393 INFO:journalctl@ceph.mon.c.smithi145.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-22T15:36:07.393 INFO:journalctl@ceph.mon.c.smithi145.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T15:36:07.393 INFO:journalctl@ceph.mon.c.smithi145.stdout: 2024-04-22T15:36:07.393 INFO:journalctl@ceph.mon.c.smithi145.stdout: ** Compaction Stats [default] ** 2024-04-22T15:36:07.393 INFO:journalctl@ceph.mon.c.smithi145.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-22T15:36:07.393 INFO:journalctl@ceph.mon.c.smithi145.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-22T15:36:07.393 INFO:journalctl@ceph.mon.c.smithi145.stdout: L0 1/0 1.53 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 542.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-22T15:36:07.393 INFO:journalctl@ceph.mon.c.smithi145.stdout: L6 1/0 11.32 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-22T15:36:07.393 INFO:journalctl@ceph.mon.c.smithi145.stdout: Sum 2/0 12.85 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 542.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-22T15:36:07.393 INFO:journalctl@ceph.mon.c.smithi145.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 542.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-22T15:36:07.394 INFO:journalctl@ceph.mon.c.smithi145.stdout: 2024-04-22T15:36:07.394 INFO:journalctl@ceph.mon.c.smithi145.stdout: ** Compaction Stats [default] ** 2024-04-22T15:36:07.394 INFO:journalctl@ceph.mon.c.smithi145.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-22T15:36:07.394 INFO:journalctl@ceph.mon.c.smithi145.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T15:36:07.394 INFO:journalctl@ceph.mon.c.smithi145.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 542.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-22T15:36:07.394 INFO:journalctl@ceph.mon.c.smithi145.stdout: 2024-04-22T15:36:07.394 INFO:journalctl@ceph.mon.c.smithi145.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-22T15:36:07.394 INFO:journalctl@ceph.mon.c.smithi145.stdout: 2024-04-22T15:36:07.394 INFO:journalctl@ceph.mon.c.smithi145.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T15:36:07.394 INFO:journalctl@ceph.mon.c.smithi145.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-04-22T15:36:07.394 INFO:journalctl@ceph.mon.c.smithi145.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-22T15:36:07.394 INFO:journalctl@ceph.mon.c.smithi145.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-22T15:36:07.394 INFO:journalctl@ceph.mon.c.smithi145.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-22T15:36:07.394 INFO:journalctl@ceph.mon.c.smithi145.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-22T15:36:07.394 INFO:journalctl@ceph.mon.c.smithi145.stdout: Cumulative compaction: 0.00 GB write, 146.40 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T15:36:07.395 INFO:journalctl@ceph.mon.c.smithi145.stdout: Interval compaction: 0.00 GB write, 146.40 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T15:36:07.395 INFO:journalctl@ceph.mon.c.smithi145.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-22T15:36:07.395 INFO:journalctl@ceph.mon.c.smithi145.stdout: Block cache BinnedLRUCache@0x5599d3ab31f0#2 capacity: 512.00 MB usage: 2.48 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-04-22T15:36:07.395 INFO:journalctl@ceph.mon.c.smithi145.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.66 KB,0.00012517%) IndexBlock(1,1.83 KB,0.000348687%) Misc(1,0.00 KB,0%) 2024-04-22T15:36:07.395 INFO:journalctl@ceph.mon.c.smithi145.stdout: 2024-04-22T15:36:07.395 INFO:journalctl@ceph.mon.c.smithi145.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-22T15:36:07.395 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: starting mon.c rank 1 at public addrs [v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0] at bind addrs [v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:36:07.395 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: mon.c@-1(???) e3 preinit fsid 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:36:07.395 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: mon.c@-1(???).mds e1 new map 2024-04-22T15:36:07.396 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: mon.c@-1(???).mds e1 print_map 2024-04-22T15:36:07.396 INFO:journalctl@ceph.mon.c.smithi145.stdout: e1 2024-04-22T15:36:07.396 INFO:journalctl@ceph.mon.c.smithi145.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-22T15:36:07.396 INFO:journalctl@ceph.mon.c.smithi145.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-22T15:36:07.396 INFO:journalctl@ceph.mon.c.smithi145.stdout: legacy client fscid: -1 2024-04-22T15:36:07.396 INFO:journalctl@ceph.mon.c.smithi145.stdout: 2024-04-22T15:36:07.396 INFO:journalctl@ceph.mon.c.smithi145.stdout: No filesystems configured 2024-04-22T15:36:07.396 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: mon.c@-1(???).osd e108 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-22T15:36:07.396 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: mon.c@-1(???).osd e108 crush map has features 432629239337189376, adjusting msgr requires 2024-04-22T15:36:07.396 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: mon.c@-1(???).osd e108 crush map has features 432629239337189376, adjusting msgr requires 2024-04-22T15:36:07.396 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: mon.c@-1(???).osd e108 crush map has features 432629239337189376, adjusting msgr requires 2024-04-22T15:36:07.396 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 ceph-mon[88162]: mon.c@-1(???).paxosservice(auth 1..27) refresh upgraded, format 0 -> 3 2024-04-22T15:36:07.396 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 bash[88143]: 38dcfa677c181defaaf0e05ccaa8446ede97daeedabb7150cb2dec4bddda3ff2 2024-04-22T15:36:07.397 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:06 smithi145 systemd[1]: Started Ceph mon.c for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:36:08.759 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:36:08.759 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[88162]: mon.a calling monitor election 2024-04-22T15:36:08.759 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[88162]: mon.c calling monitor election 2024-04-22T15:36:08.760 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[88162]: mon.b calling monitor election 2024-04-22T15:36:08.760 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:36:08.760 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:36:08.760 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[88162]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-22T15:36:08.760 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[88162]: monmap epoch 4 2024-04-22T15:36:08.760 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[88162]: fsid 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:36:08.760 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[88162]: last_changed 2024-04-22T15:36:07.617322+0000 2024-04-22T15:36:08.760 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[88162]: created 2024-04-22T15:04:30.924274+0000 2024-04-22T15:36:08.760 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[88162]: min_mon_release 19 (squid) 2024-04-22T15:36:08.760 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[88162]: election_strategy: 1 2024-04-22T15:36:08.760 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[88162]: 0: [v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0] mon.a 2024-04-22T15:36:08.760 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[88162]: 1: [v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0] mon.c 2024-04-22T15:36:08.761 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[88162]: 2: [v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0] mon.b 2024-04-22T15:36:08.761 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[88162]: fsmap 2024-04-22T15:36:08.761 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[88162]: osdmap e108: 8 total, 8 up, 8 in 2024-04-22T15:36:08.761 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[88162]: mgrmap e49: y(active, since 18s), standbys: x 2024-04-22T15:36:08.761 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[88162]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:36:08.761 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[88162]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:36:08.761 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[88162]: daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state 2024-04-22T15:36:08.761 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:08.761 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:08.761 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:36:08.761 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[88162]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 970 B/s rd, 0 op/s 2024-04-22T15:36:08.762 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:36:08.762 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[82523]: mon.a calling monitor election 2024-04-22T15:36:08.762 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[82523]: mon.c calling monitor election 2024-04-22T15:36:08.762 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[82523]: mon.b calling monitor election 2024-04-22T15:36:08.762 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:36:08.762 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:36:08.762 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[82523]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-22T15:36:08.762 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[82523]: monmap epoch 4 2024-04-22T15:36:08.762 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[82523]: fsid 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:36:08.762 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[82523]: last_changed 2024-04-22T15:36:07.617322+0000 2024-04-22T15:36:08.762 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[82523]: created 2024-04-22T15:04:30.924274+0000 2024-04-22T15:36:08.763 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[82523]: min_mon_release 19 (squid) 2024-04-22T15:36:08.763 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[82523]: election_strategy: 1 2024-04-22T15:36:08.763 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[82523]: 0: [v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0] mon.a 2024-04-22T15:36:08.763 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[82523]: 1: [v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0] mon.c 2024-04-22T15:36:08.763 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[82523]: 2: [v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0] mon.b 2024-04-22T15:36:08.763 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[82523]: fsmap 2024-04-22T15:36:08.763 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[82523]: osdmap e108: 8 total, 8 up, 8 in 2024-04-22T15:36:08.763 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[82523]: mgrmap e49: y(active, since 18s), standbys: x 2024-04-22T15:36:08.763 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[82523]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:36:08.763 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[82523]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:36:08.763 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[82523]: daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state 2024-04-22T15:36:08.763 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:08.764 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:08.764 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:36:08.764 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:08 smithi145 ceph-mon[82523]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 970 B/s rd, 0 op/s 2024-04-22T15:36:08.928 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:08 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T15:36:08.928 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:08 smithi172 ceph-mon[65307]: mon.a calling monitor election 2024-04-22T15:36:08.928 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:08 smithi172 ceph-mon[65307]: mon.c calling monitor election 2024-04-22T15:36:08.928 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:08 smithi172 ceph-mon[65307]: mon.b calling monitor election 2024-04-22T15:36:08.928 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:08 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T15:36:08.928 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:08 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T15:36:08.929 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:08 smithi172 ceph-mon[65307]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-22T15:36:08.929 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:08 smithi172 ceph-mon[65307]: monmap epoch 4 2024-04-22T15:36:08.929 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:08 smithi172 ceph-mon[65307]: fsid 869afcfa-00b9-11ef-bc93-c7b262605968 2024-04-22T15:36:08.929 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:08 smithi172 ceph-mon[65307]: last_changed 2024-04-22T15:36:07.617322+0000 2024-04-22T15:36:08.929 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:08 smithi172 ceph-mon[65307]: created 2024-04-22T15:04:30.924274+0000 2024-04-22T15:36:08.929 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:08 smithi172 ceph-mon[65307]: min_mon_release 19 (squid) 2024-04-22T15:36:08.929 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:08 smithi172 ceph-mon[65307]: election_strategy: 1 2024-04-22T15:36:08.929 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:08 smithi172 ceph-mon[65307]: 0: [v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0] mon.a 2024-04-22T15:36:08.929 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:08 smithi172 ceph-mon[65307]: 1: [v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0] mon.c 2024-04-22T15:36:08.929 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:08 smithi172 ceph-mon[65307]: 2: [v2:172.21.15.172:3300/0,v1:172.21.15.172:6789/0] mon.b 2024-04-22T15:36:08.929 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:08 smithi172 ceph-mon[65307]: fsmap 2024-04-22T15:36:08.929 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:08 smithi172 ceph-mon[65307]: osdmap e108: 8 total, 8 up, 8 in 2024-04-22T15:36:08.929 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:08 smithi172 ceph-mon[65307]: mgrmap e49: y(active, since 18s), standbys: x 2024-04-22T15:36:08.930 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:08 smithi172 ceph-mon[65307]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:36:08.930 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:08 smithi172 ceph-mon[65307]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:36:08.930 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:08 smithi172 ceph-mon[65307]: daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state 2024-04-22T15:36:08.930 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:08 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:08.930 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:08 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:08.930 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:08 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:36:08.930 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:08 smithi172 ceph-mon[65307]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 970 B/s rd, 0 op/s 2024-04-22T15:36:11.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:10 smithi145 ceph-mon[82523]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 970 B/s rd, 0 op/s 2024-04-22T15:36:11.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:10 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:36:11.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:10 smithi145 ceph-mon[88162]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 970 B/s rd, 0 op/s 2024-04-22T15:36:11.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:10 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:36:11.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:10 smithi172 ceph-mon[65307]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 970 B/s rd, 0 op/s 2024-04-22T15:36:11.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:10 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:36:12.372 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:11 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:36:11.920+0000 7f17e84c2700 -1 mgr.server handle_report got status from non-daemon mon.c 2024-04-22T15:36:13.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:13 smithi145 ceph-mon[82523]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:36:13.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:13 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:13.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:13 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:13.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:13 smithi145 ceph-mon[88162]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:36:13.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:13 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:13.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:13 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:13.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:13 smithi172 ceph-mon[65307]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:36:13.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:13 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:13.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:13 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:14.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:14 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:14.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:14 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:14.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:14 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:14.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:14 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:14.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:14 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:14.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:14 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:15.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:15 smithi145 ceph-mon[88162]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:36:15.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:15 smithi145 ceph-mon[82523]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:36:15.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:36:15] "GET /metrics HTTP/1.1" 200 38019 "" "Prometheus/2.43.0" 2024-04-22T15:36:15.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:15 smithi172 ceph-mon[65307]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:36:17.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:17 smithi172 ceph-mon[65307]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:36:17.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:17 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:17.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:17 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:17.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:17 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:17.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:17 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:36:17.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:17 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:17.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:17 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:36:17.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:17 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:36:17.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:17 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:17.500 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:17 smithi145 ceph-mon[82523]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:36:17.500 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:17 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:17.500 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:17 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:17.501 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:17 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:17.501 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:17 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:36:17.501 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:17 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:17.501 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:17 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:36:17.501 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:17 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:36:17.501 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:17 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:17.501 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:17 smithi145 ceph-mon[88162]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:36:17.501 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:17 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:17.501 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:17 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:17.501 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:17 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:17.501 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:17 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:36:17.501 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:17 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:17.502 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:17 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:36:17.502 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:17 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:36:17.502 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:17 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:18.436 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:18 smithi172 ceph-mon[65307]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 928 B/s rd, 0 op/s 2024-04-22T15:36:18.436 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:18 smithi172 ceph-mon[65307]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:36:18.436 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:18 smithi172 ceph-mon[65307]: Reconfiguring mon.a (monmap changed)... 2024-04-22T15:36:18.436 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:18 smithi172 ceph-mon[65307]: Reconfiguring daemon mon.a on smithi145 2024-04-22T15:36:18.525 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:18 smithi145 ceph-mon[88162]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 928 B/s rd, 0 op/s 2024-04-22T15:36:18.525 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:18 smithi145 ceph-mon[88162]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:36:18.525 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:18 smithi145 ceph-mon[88162]: Reconfiguring mon.a (monmap changed)... 2024-04-22T15:36:18.525 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:18 smithi145 ceph-mon[88162]: Reconfiguring daemon mon.a on smithi145 2024-04-22T15:36:18.525 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:18 smithi145 ceph-mon[82523]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 928 B/s rd, 0 op/s 2024-04-22T15:36:18.525 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:18 smithi145 ceph-mon[82523]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:36:18.525 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:18 smithi145 ceph-mon[82523]: Reconfiguring mon.a (monmap changed)... 2024-04-22T15:36:18.525 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:18 smithi145 ceph-mon[82523]: Reconfiguring daemon mon.a on smithi145 2024-04-22T15:36:19.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:19 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:19.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:19 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:19.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:19 smithi145 ceph-mon[82523]: Reconfiguring mgr.y (monmap changed)... 2024-04-22T15:36:19.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:19 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:36:19.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:19 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:36:19.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:19 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:19.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:19 smithi145 ceph-mon[82523]: Reconfiguring daemon mgr.y on smithi145 2024-04-22T15:36:19.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:19 smithi145 ceph-mon[82523]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:36:19.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:19 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:36:19.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:19 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:19.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:19 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:19.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:19 smithi145 ceph-mon[88162]: Reconfiguring mgr.y (monmap changed)... 2024-04-22T15:36:19.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:19 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:36:19.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:19 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:36:19.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:19 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:19.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:19 smithi145 ceph-mon[88162]: Reconfiguring daemon mgr.y on smithi145 2024-04-22T15:36:19.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:19 smithi145 ceph-mon[88162]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:36:19.874 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:19 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:36:19.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:19 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:19.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:19 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:19.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:19 smithi172 ceph-mon[65307]: Reconfiguring mgr.y (monmap changed)... 2024-04-22T15:36:19.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:19 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:36:19.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:19 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:36:19.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:19 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:19.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:19 smithi172 ceph-mon[65307]: Reconfiguring daemon mgr.y on smithi145 2024-04-22T15:36:19.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:19 smithi172 ceph-mon[65307]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:36:19.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:19 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:36:21.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:21 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:21.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:21 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:21.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:21 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:36:21.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:21 smithi145 ceph-mon[88162]: Reconfiguring mon.c (monmap changed)... 2024-04-22T15:36:21.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:21 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:36:21.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:21 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:36:21.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:21 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:21.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:21 smithi145 ceph-mon[88162]: Reconfiguring daemon mon.c on smithi145 2024-04-22T15:36:21.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:21 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:21.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:21 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:21.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:21 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:36:21.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:21 smithi145 ceph-mon[82523]: Reconfiguring mon.c (monmap changed)... 2024-04-22T15:36:21.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:21 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:36:21.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:21 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:36:21.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:21 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:21.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:21 smithi145 ceph-mon[82523]: Reconfiguring daemon mon.c on smithi145 2024-04-22T15:36:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:21 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:21 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:21 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:36:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:21 smithi172 ceph-mon[65307]: Reconfiguring mon.c (monmap changed)... 2024-04-22T15:36:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:21 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:36:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:21 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:36:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:21 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:21 smithi172 ceph-mon[65307]: Reconfiguring daemon mon.c on smithi145 2024-04-22T15:36:22.611 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:22 smithi145 ceph-mon[88162]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:36:22.611 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:22 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:22.611 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:22 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:22.611 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:22 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T15:36:22.611 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:22 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:22.611 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:22 smithi145 ceph-mon[82523]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:36:22.611 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:22 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:22.611 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:22 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:22.611 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:22 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T15:36:22.611 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:22 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:22.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:22 smithi172 ceph-mon[65307]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:36:22.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:22 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:22.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:22 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:22.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:22 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T15:36:22.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:22 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:23.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:23 smithi145 ceph-mon[82523]: Reconfiguring osd.0 (monmap changed)... 2024-04-22T15:36:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:23 smithi145 ceph-mon[82523]: Reconfiguring daemon osd.0 on smithi145 2024-04-22T15:36:23.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:23 smithi145 ceph-mon[88162]: Reconfiguring osd.0 (monmap changed)... 2024-04-22T15:36:23.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:23 smithi145 ceph-mon[88162]: Reconfiguring daemon osd.0 on smithi145 2024-04-22T15:36:23.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:23 smithi172 ceph-mon[65307]: Reconfiguring osd.0 (monmap changed)... 2024-04-22T15:36:23.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:23 smithi172 ceph-mon[65307]: Reconfiguring daemon osd.0 on smithi145 2024-04-22T15:36:24.554 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:24 smithi145 ceph-mon[88162]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:36:24.554 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:24 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:24.554 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:24 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:24.554 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:24 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T15:36:24.554 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:24 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:24.554 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:24 smithi145 ceph-mon[82523]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:36:24.554 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:24 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:24.554 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:24 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:24.555 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:24 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T15:36:24.555 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:24 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:24.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:24 smithi172 ceph-mon[65307]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:36:24.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:24 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:24.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:24 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:24.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:24 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T15:36:24.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:24 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:25.507 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:25 smithi145 ceph-mon[82523]: Reconfiguring osd.1 (monmap changed)... 2024-04-22T15:36:25.507 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:25 smithi145 ceph-mon[82523]: Reconfiguring daemon osd.1 on smithi145 2024-04-22T15:36:25.508 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:25 smithi145 ceph-mon[88162]: Reconfiguring osd.1 (monmap changed)... 2024-04-22T15:36:25.508 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:25 smithi145 ceph-mon[88162]: Reconfiguring daemon osd.1 on smithi145 2024-04-22T15:36:25.508 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:36:25] "GET /metrics HTTP/1.1" 200 38031 "" "Prometheus/2.43.0" 2024-04-22T15:36:25.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:25 smithi172 ceph-mon[65307]: Reconfiguring osd.1 (monmap changed)... 2024-04-22T15:36:25.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:25 smithi172 ceph-mon[65307]: Reconfiguring daemon osd.1 on smithi145 2024-04-22T15:36:26.536 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:26 smithi145 ceph-mon[88162]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 567 B/s rd, 0 op/s 2024-04-22T15:36:26.536 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:26 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:26.536 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:26 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:26.536 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:26 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T15:36:26.536 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:26 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:26.536 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:26 smithi145 ceph-mon[82523]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 567 B/s rd, 0 op/s 2024-04-22T15:36:26.536 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:26 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:26.536 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:26 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:26.536 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:26 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T15:36:26.537 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:26 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:26.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:26 smithi172 ceph-mon[65307]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 567 B/s rd, 0 op/s 2024-04-22T15:36:26.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:26 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:26.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:26 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:26.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:26 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T15:36:26.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:26 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:27.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:27 smithi145 ceph-mon[88162]: Reconfiguring osd.2 (monmap changed)... 2024-04-22T15:36:27.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:27 smithi145 ceph-mon[88162]: Reconfiguring daemon osd.2 on smithi145 2024-04-22T15:36:27.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:27 smithi145 ceph-mon[82523]: Reconfiguring osd.2 (monmap changed)... 2024-04-22T15:36:27.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:27 smithi145 ceph-mon[82523]: Reconfiguring daemon osd.2 on smithi145 2024-04-22T15:36:27.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:27 smithi172 ceph-mon[65307]: Reconfiguring osd.2 (monmap changed)... 2024-04-22T15:36:27.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:27 smithi172 ceph-mon[65307]: Reconfiguring daemon osd.2 on smithi145 2024-04-22T15:36:28.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:28 smithi145 ceph-mon[88162]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:36:28.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:28 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:28.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:28 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:28.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:28 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T15:36:28.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:28 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:28.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:28 smithi145 ceph-mon[82523]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:36:28.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:28 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:28.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:28 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:28.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:28 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T15:36:28.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:28 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:28.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:28 smithi172 ceph-mon[65307]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:36:28.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:28 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:28.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:28 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:28.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:28 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T15:36:28.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:28 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:29.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:29 smithi145 ceph-mon[82523]: Reconfiguring osd.3 (monmap changed)... 2024-04-22T15:36:29.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:29 smithi145 ceph-mon[82523]: Reconfiguring daemon osd.3 on smithi145 2024-04-22T15:36:29.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:29 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:29.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:29 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:29.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:29 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi145.ygrqmi", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T15:36:29.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:29 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:29.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:29 smithi145 ceph-mon[88162]: Reconfiguring osd.3 (monmap changed)... 2024-04-22T15:36:29.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:29 smithi145 ceph-mon[88162]: Reconfiguring daemon osd.3 on smithi145 2024-04-22T15:36:29.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:29 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:29.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:29 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:29.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:29 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi145.ygrqmi", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T15:36:29.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:29 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:29.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:29 smithi172 ceph-mon[65307]: Reconfiguring osd.3 (monmap changed)... 2024-04-22T15:36:29.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:29 smithi172 ceph-mon[65307]: Reconfiguring daemon osd.3 on smithi145 2024-04-22T15:36:29.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:29 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:29.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:29 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:29.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:29 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi145.ygrqmi", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T15:36:29.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:29 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:30.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:30 smithi145 ceph-mon[88162]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:36:30.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:30 smithi145 ceph-mon[88162]: Reconfiguring rgw.foo.smithi145.ygrqmi (monmap changed)... 2024-04-22T15:36:30.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:30 smithi145 ceph-mon[88162]: Reconfiguring daemon rgw.foo.smithi145.ygrqmi on smithi145 2024-04-22T15:36:30.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:30 smithi145 ceph-mon[82523]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:36:30.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:30 smithi145 ceph-mon[82523]: Reconfiguring rgw.foo.smithi145.ygrqmi (monmap changed)... 2024-04-22T15:36:30.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:30 smithi145 ceph-mon[82523]: Reconfiguring daemon rgw.foo.smithi145.ygrqmi on smithi145 2024-04-22T15:36:30.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:30 smithi172 ceph-mon[65307]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:36:30.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:30 smithi172 ceph-mon[65307]: Reconfiguring rgw.foo.smithi145.ygrqmi (monmap changed)... 2024-04-22T15:36:30.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:30 smithi172 ceph-mon[65307]: Reconfiguring daemon rgw.foo.smithi145.ygrqmi on smithi145 2024-04-22T15:36:31.516 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:31 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:36:31.517 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:31 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:31.517 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:31 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:31.517 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:31 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:36:31.517 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:31 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:31.517 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:31 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:36:31.517 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:31 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:31.517 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:31 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:31.517 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:31 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:36:31.517 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:31 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:31 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:36:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:31 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:31 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:31 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:36:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:31 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:32.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-mon[82523]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:36:32.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-mon[82523]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:36:32.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-mon[82523]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:36:32.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-mon[88162]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:36:32.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-mon[88162]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:36:32.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-mon[88162]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:36:32.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:32 smithi172 ceph-mon[65307]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:36:32.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:32 smithi172 ceph-mon[65307]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:36:32.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:32 smithi172 ceph-mon[65307]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:36:32.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:36:32.650+0000 7f17b18ae700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:36:32.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:36:32.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:36:32.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:36:32.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:36:32.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:32.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:32.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:36:32.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:36:32.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:36:32.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:36:32.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:36:32.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:36:32.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:36:32.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:36:32.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:36:32.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:36:32.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:36:32.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:36:32.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:32.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:32.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:36:32.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:36:32.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:36:32.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:36:32.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:36:32.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:36:32.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:36:32.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:36:32.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:36:32.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:36:32.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:36:32.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:36:32.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:36:32.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:36:32.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:36:32.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:36:32.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:36:32.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:36:32.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:36:32.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:36:32.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:36:32.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:36:32.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:32.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:32.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:36:32.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:36:32.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:36:32.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:36:32.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:36:32.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:36:32.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:36:32.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:36:32.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:36:32.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:36:32.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:36:32.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:36:32.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:32.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:33.493 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:33 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:36:33.493 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:33 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:36:33.493 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:33 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:33.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:33 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:36:33.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:33 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:36:33.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:33 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:33.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:33 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T15:36:33.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:33 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T15:36:33.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:33 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:34.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:34 smithi172 ceph-mon[65307]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:36:34.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:36:34.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:36:34.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:36:34.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:36:34.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:34.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:34.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:36:34.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:36:34.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:36:34.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:36:34.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:36:34.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:36:34.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:36:34.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:36:34.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:36:34.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:36:34.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:36:34.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:36:34.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:34.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:34.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:36:34.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:36:34.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:36:34.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:36:34.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:36:34.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:36:34.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:36:34.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:36:34.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:36:34.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:36:34.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:36:34.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:36:34.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:36:34.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:36:34.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:36:34.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:36:34.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:36:34.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:36:34.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:36:34.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:36:34.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:36:34.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:36:34.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:34.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:34.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:36:34.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:36:34.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:36:34.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:36:34.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:36:34.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:36:34.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:36:34.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:36:34.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:36:34.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:36:34.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:36:34.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:36:34.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:34.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:34.472 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:34 smithi172 ceph-mon[65307]: Reconfiguring mon.b (monmap changed)... 2024-04-22T15:36:34.472 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:34 smithi172 ceph-mon[65307]: Reconfiguring daemon mon.b on smithi172 2024-04-22T15:36:34.472 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:34 smithi172 ceph-mon[65307]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:36:34.472 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:34.472 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:34.472 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:36:34.472 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:36:34.472 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:34.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:34 smithi145 ceph-mon[82523]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:36:34.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:36:34.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:36:34.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:36:34.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:36:34.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:34.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:34.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:36:34.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:36:34.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:36:34.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:36:34.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:36:34.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:36:34.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:36:34.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:36:34.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:36:34.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:36:34.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:36:34.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:36:34.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:34.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:34.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:36:34.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:36:34.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:36:34.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:36:34.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:36:34.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:36:34.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:36:34.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:36:34.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:36:34.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:36:34.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:36:34.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:36:34.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:36:34.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:36:34.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:36:34.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:36:34.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:36:34.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:36:34.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:36:34.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:36:34.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:36:34.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:36:34.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:34.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:34.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:36:34.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:36:34.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:36:34.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:36:34.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:36:34.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:36:34.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:36:34.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:36:34.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:36:34.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:34 smithi145 ceph-mon[88162]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:36:34.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:36:34.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:36:34.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:36:34.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:36:34.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:34.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:34.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:36:34.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:36:34.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:36:34.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:36:34.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:36:34.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:36:34.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:36:34.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:36:34.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:36:34.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:36:34.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:36:34.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:36:34.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:34.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:34.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:36:34.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:36:34.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:36:34.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:36:34.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:36:34.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:36:34.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:36:34.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:36:34.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:36:34.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:36:34.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:36:34.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:36:34.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:36:34.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:36:34.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:36:34.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:36:34.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:36:34.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:36:34.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:36:34.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:36:34.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:36:34.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:36:34.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:34.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:34.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:36:34.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:36:34.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:36:34.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:36:34.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:36:34.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:36:34.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:36:34.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:36:34.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:36:34.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:36:34.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:36:34.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:36:34.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:34.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:34.631 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:34 smithi145 ceph-mon[82523]: Reconfiguring mon.b (monmap changed)... 2024-04-22T15:36:34.631 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:34 smithi145 ceph-mon[82523]: Reconfiguring daemon mon.b on smithi172 2024-04-22T15:36:34.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:34 smithi145 ceph-mon[82523]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:36:34.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:34.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:34.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:36:34.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:36:34.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:34.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:36:34.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:36:34.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:36:34.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:34.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:34.632 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:34 smithi145 ceph-mon[88162]: Reconfiguring mon.b (monmap changed)... 2024-04-22T15:36:34.633 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:34 smithi145 ceph-mon[88162]: Reconfiguring daemon mon.b on smithi172 2024-04-22T15:36:34.633 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:34 smithi145 ceph-mon[88162]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:36:34.633 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:34.633 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:34.633 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T15:36:34.633 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T15:36:34.633 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:35.429 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:35 smithi172 ceph-mon[65307]: Reconfiguring mgr.x (monmap changed)... 2024-04-22T15:36:35.429 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:35 smithi172 ceph-mon[65307]: Reconfiguring daemon mgr.x on smithi172 2024-04-22T15:36:35.429 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:36:35.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:35 smithi145 ceph-mon[82523]: Reconfiguring mgr.x (monmap changed)... 2024-04-22T15:36:35.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:35 smithi145 ceph-mon[82523]: Reconfiguring daemon mgr.x on smithi172 2024-04-22T15:36:35.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:36:35.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:35 smithi145 ceph-mon[88162]: Reconfiguring mgr.x (monmap changed)... 2024-04-22T15:36:35.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:35 smithi145 ceph-mon[88162]: Reconfiguring daemon mgr.x on smithi172 2024-04-22T15:36:35.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:36:35.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:36:35] "GET /metrics HTTP/1.1" 200 38027 "" "Prometheus/2.43.0" 2024-04-22T15:36:36.177 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:36:36.541 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:36 smithi172 ceph-mon[65307]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:36:36.541 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:36 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:36.542 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:36 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:36.542 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:36 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T15:36:36.542 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:36 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:36.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:36 smithi145 ceph-mon[82523]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:36:36.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:36 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:36.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:36 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:36.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:36 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T15:36:36.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:36 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:36.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:36 smithi145 ceph-mon[88162]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:36:36.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:36 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:36.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:36 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:36.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:36 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T15:36:36.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:36 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:36.830 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:36:36.830 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (3m) 24s ago 25m 16.7M - 0.25.0 c8568f914cd2 1c58ccf37ffd 2024-04-22T15:36:36.831 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (3m) 43s ago 25m 79.5M - 9.4.12 2bacad6d85d8 2f825d15ab10 2024-04-22T15:36:36.831 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 unknown 24s ago 24m 60.5M - 3.5 e1d6a67b021e 9704631ffa0e 2024-04-22T15:36:36.831 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443,9283,8765 running (4m) 43s ago 30m 448M - 19.0.0-3223-g5883d494 3d983c26e278 e7500ef71604 2024-04-22T15:36:36.831 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (14m) 24s ago 31m 496M - 19.0.0-3223-g5883d494 3d983c26e278 b2de97566cf4 2024-04-22T15:36:36.831 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (57s) 24s ago 32m 39.4M 2048M 19.0.0-3223-g5883d494 3d983c26e278 7bfdc11120f4 2024-04-22T15:36:36.831 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (108s) 43s ago 30m 39.4M 2048M 19.0.0-3223-g5883d494 3d983c26e278 0a360b08a6d6 2024-04-22T15:36:36.831 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (30s) 24s ago 30m 19.9M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38dcfa677c18 2024-04-22T15:36:36.831 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (4m) 24s ago 26m 13.1M - 1.5.0 0da6a335fe13 8c75c26fec05 2024-04-22T15:36:36.831 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (4m) 43s ago 26m 15.9M - 1.5.0 0da6a335fe13 8fd2949e2eaf 2024-04-22T15:36:36.831 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (30m) 24s ago 30m 57.2M 2992M 17.2.0 e1d6a67b021e ea9cdc639916 2024-04-22T15:36:36.831 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (29m) 24s ago 29m 55.4M 2992M 17.2.0 e1d6a67b021e 3b9b37a8781f 2024-04-22T15:36:36.831 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (29m) 24s ago 29m 51.1M 2992M 17.2.0 e1d6a67b021e d9502dded8f8 2024-04-22T15:36:36.831 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (28m) 24s ago 28m 53.1M 2992M 17.2.0 e1d6a67b021e 8c72f30e54fc 2024-04-22T15:36:36.831 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (28m) 43s ago 28m 57.8M 3248M 17.2.0 e1d6a67b021e 80001f97ed35 2024-04-22T15:36:36.831 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (28m) 43s ago 28m 54.2M 3248M 17.2.0 e1d6a67b021e ec3ab1335b5c 2024-04-22T15:36:36.831 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (27m) 43s ago 27m 54.9M 3248M 17.2.0 e1d6a67b021e 45d475edf1ca 2024-04-22T15:36:36.831 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (27m) 43s ago 27m 53.0M 3248M 17.2.0 e1d6a67b021e cc9e1d9e7916 2024-04-22T15:36:36.832 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (3m) 43s ago 26m 63.9M - 2.43.0 a07b618ecd1d 51cf4df6e8d4 2024-04-22T15:36:36.832 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (25m) 24s ago 24m 79.2M - 17.2.0 e1d6a67b021e c90a636c947e 2024-04-22T15:36:36.832 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (24m) 43s ago 24m 76.5M - 17.2.0 e1d6a67b021e 20ac46e9bfda 2024-04-22T15:36:37.233 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:36:37.233 INFO:teuthology.orchestra.run.smithi145.stdout: "mon": { 2024-04-22T15:36:37.233 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 3 2024-04-22T15:36:37.234 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:36:37.234 INFO:teuthology.orchestra.run.smithi145.stdout: "mgr": { 2024-04-22T15:36:37.234 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:36:37.234 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:36:37.234 INFO:teuthology.orchestra.run.smithi145.stdout: "osd": { 2024-04-22T15:36:37.234 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T15:36:37.234 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:36:37.234 INFO:teuthology.orchestra.run.smithi145.stdout: "rgw": { 2024-04-22T15:36:37.234 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:36:37.234 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:36:37.234 INFO:teuthology.orchestra.run.smithi145.stdout: "overall": { 2024-04-22T15:36:37.234 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-04-22T15:36:37.234 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 5 2024-04-22T15:36:37.234 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:36:37.235 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:36:37.576 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:36:37.576 INFO:teuthology.orchestra.run.smithi145.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:36:37.576 INFO:teuthology.orchestra.run.smithi145.stdout: "in_progress": true, 2024-04-22T15:36:37.576 INFO:teuthology.orchestra.run.smithi145.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi145", 2024-04-22T15:36:37.576 INFO:teuthology.orchestra.run.smithi145.stdout: "services_complete": [ 2024-04-22T15:36:37.576 INFO:teuthology.orchestra.run.smithi145.stdout: "mon" 2024-04-22T15:36:37.576 INFO:teuthology.orchestra.run.smithi145.stdout: ], 2024-04-22T15:36:37.576 INFO:teuthology.orchestra.run.smithi145.stdout: "progress": "2/2 daemons upgraded", 2024-04-22T15:36:37.576 INFO:teuthology.orchestra.run.smithi145.stdout: "message": "Currently upgrading mon daemons", 2024-04-22T15:36:37.576 INFO:teuthology.orchestra.run.smithi145.stdout: "is_paused": false 2024-04-22T15:36:37.576 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:36:37.577 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:37 smithi145 ceph-mon[88162]: Reconfiguring osd.4 (monmap changed)... 2024-04-22T15:36:37.577 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:37 smithi145 ceph-mon[88162]: Reconfiguring daemon osd.4 on smithi172 2024-04-22T15:36:37.577 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:37 smithi145 ceph-mon[88162]: from='client.44166 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:36:37.577 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:37 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/2241464943' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:37.577 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:37 smithi145 ceph-mon[82523]: Reconfiguring osd.4 (monmap changed)... 2024-04-22T15:36:37.577 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:37 smithi145 ceph-mon[82523]: Reconfiguring daemon osd.4 on smithi172 2024-04-22T15:36:37.577 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:37 smithi145 ceph-mon[82523]: from='client.44166 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:36:37.577 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:37 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/2241464943' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:37.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:37 smithi172 ceph-mon[65307]: Reconfiguring osd.4 (monmap changed)... 2024-04-22T15:36:37.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:37 smithi172 ceph-mon[65307]: Reconfiguring daemon osd.4 on smithi172 2024-04-22T15:36:37.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:37 smithi172 ceph-mon[65307]: from='client.44166 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:36:37.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:37 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/2241464943' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:38.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:38 smithi145 ceph-mon[88162]: from='client.44172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:36:38.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:38 smithi145 ceph-mon[88162]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:36:38.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:38 smithi145 ceph-mon[88162]: from='client.54115 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:36:38.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:38 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:38.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:38 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:38.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:38 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T15:36:38.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:38 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:38.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:38 smithi145 ceph-mon[82523]: from='client.44172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:36:38.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:38 smithi145 ceph-mon[82523]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:36:38.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:38 smithi145 ceph-mon[82523]: from='client.54115 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:36:38.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:38 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:38.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:38 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:38.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:38 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T15:36:38.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:38 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:38.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:38 smithi172 ceph-mon[65307]: from='client.44172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:36:38.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:38 smithi172 ceph-mon[65307]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:36:38.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:38 smithi172 ceph-mon[65307]: from='client.54115 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:36:38.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:38 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:38.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:38 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:38.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:38 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T15:36:38.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:38 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:39.601 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:39 smithi172 ceph-mon[65307]: Reconfiguring osd.5 (monmap changed)... 2024-04-22T15:36:39.601 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:39 smithi172 ceph-mon[65307]: Reconfiguring daemon osd.5 on smithi172 2024-04-22T15:36:39.602 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:39 smithi172 ceph-mon[65307]: from='client.54127 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:36:39.602 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:39 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:39.602 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:39 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:39.602 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:39 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T15:36:39.602 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:39 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:39.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:39 smithi145 ceph-mon[82523]: Reconfiguring osd.5 (monmap changed)... 2024-04-22T15:36:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:39 smithi145 ceph-mon[82523]: Reconfiguring daemon osd.5 on smithi172 2024-04-22T15:36:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:39 smithi145 ceph-mon[82523]: from='client.54127 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:36:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:39 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:39 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:39 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T15:36:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:39 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:39.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:39 smithi145 ceph-mon[88162]: Reconfiguring osd.5 (monmap changed)... 2024-04-22T15:36:39.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:39 smithi145 ceph-mon[88162]: Reconfiguring daemon osd.5 on smithi172 2024-04-22T15:36:39.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:39 smithi145 ceph-mon[88162]: from='client.54127 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:36:39.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:39 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:39.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:39 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:39.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:39 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T15:36:39.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:39 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:40.404 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:40 smithi172 ceph-mon[65307]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:36:40.404 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:40 smithi172 ceph-mon[65307]: Reconfiguring osd.6 (monmap changed)... 2024-04-22T15:36:40.404 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:40 smithi172 ceph-mon[65307]: Reconfiguring daemon osd.6 on smithi172 2024-04-22T15:36:40.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:40 smithi145 ceph-mon[82523]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:36:40.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:40 smithi145 ceph-mon[82523]: Reconfiguring osd.6 (monmap changed)... 2024-04-22T15:36:40.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:40 smithi145 ceph-mon[82523]: Reconfiguring daemon osd.6 on smithi172 2024-04-22T15:36:40.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:40 smithi145 ceph-mon[88162]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:36:40.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:40 smithi145 ceph-mon[88162]: Reconfiguring osd.6 (monmap changed)... 2024-04-22T15:36:40.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:40 smithi145 ceph-mon[88162]: Reconfiguring daemon osd.6 on smithi172 2024-04-22T15:36:41.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:41 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:36:41.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:41 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:41.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:41 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:41.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:41 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T15:36:41.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:41 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:41.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:41 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:36:41.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:41 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:41.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:41 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:41.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:41 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T15:36:41.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:41 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:41.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:41 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:36:41.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:41 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:41.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:41 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:41.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:41 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T15:36:41.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:41 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:42.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:42 smithi145 ceph-mon[88162]: Reconfiguring osd.7 (monmap changed)... 2024-04-22T15:36:42.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:42 smithi145 ceph-mon[88162]: Reconfiguring daemon osd.7 on smithi172 2024-04-22T15:36:42.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:42 smithi145 ceph-mon[88162]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:36:42.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:42 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:42.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:42 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:42.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:42 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.glxwwa", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T15:36:42.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:42 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:42.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:42 smithi145 ceph-mon[82523]: Reconfiguring osd.7 (monmap changed)... 2024-04-22T15:36:42.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:42 smithi145 ceph-mon[82523]: Reconfiguring daemon osd.7 on smithi172 2024-04-22T15:36:42.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:42 smithi145 ceph-mon[82523]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:36:42.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:42 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:42.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:42 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:42.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:42 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.glxwwa", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T15:36:42.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:42 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:42.642 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:42 smithi172 ceph-mon[65307]: Reconfiguring osd.7 (monmap changed)... 2024-04-22T15:36:42.642 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:42 smithi172 ceph-mon[65307]: Reconfiguring daemon osd.7 on smithi172 2024-04-22T15:36:42.642 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:42 smithi172 ceph-mon[65307]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:36:42.642 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:42 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:42.642 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:42 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:42.642 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:42 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.glxwwa", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T15:36:42.642 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:42 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:43.570 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:43 smithi172 ceph-mon[65307]: Reconfiguring rgw.foo.smithi172.glxwwa (monmap changed)... 2024-04-22T15:36:43.571 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:43 smithi172 ceph-mon[65307]: Reconfiguring daemon rgw.foo.smithi172.glxwwa on smithi172 2024-04-22T15:36:43.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:43 smithi145 ceph-mon[82523]: Reconfiguring rgw.foo.smithi172.glxwwa (monmap changed)... 2024-04-22T15:36:43.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:43 smithi145 ceph-mon[82523]: Reconfiguring daemon rgw.foo.smithi172.glxwwa on smithi172 2024-04-22T15:36:43.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:43 smithi145 ceph-mon[88162]: Reconfiguring rgw.foo.smithi172.glxwwa (monmap changed)... 2024-04-22T15:36:43.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:43 smithi145 ceph-mon[88162]: Reconfiguring daemon rgw.foo.smithi172.glxwwa on smithi172 2024-04-22T15:36:44.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:36:44.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:44.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:44.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:36:44.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:44.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T15:36:44.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-22T15:36:44.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T15:36:44.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-22T15:36:44.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:44.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:44.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:44.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T15:36:44.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T15:36:44.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:44.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:44.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:44.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T15:36:44.628 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:36:44.628 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:44.628 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:44.628 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:36:44.628 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.628 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.628 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:44.628 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T15:36:44.628 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-22T15:36:44.628 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T15:36:44.628 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-22T15:36:44.628 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.628 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:44.629 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.629 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:44.629 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.629 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:44.629 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.629 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T15:36:44.629 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T15:36:44.629 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.629 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:44.629 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.629 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:44.629 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.629 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:44.629 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.630 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T15:36:44.630 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T15:36:44.630 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.630 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T15:36:44.630 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T15:36:44.630 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.630 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T15:36:44.630 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T15:36:44.630 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.630 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T15:36:44.630 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T15:36:44.631 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.631 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T15:36:44.631 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T15:36:44.631 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.631 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T15:36:44.631 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T15:36:44.631 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T15:36:44.631 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T15:36:44.631 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:36:44.631 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T15:36:44.631 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T15:36:44.631 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T15:36:44.631 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T15:36:44.631 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T15:36:44.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T15:36:44.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T15:36:44.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T15:36:44.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T15:36:44.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T15:36:44.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T15:36:44.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T15:36:44.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T15:36:44.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T15:36:44.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T15:36:44.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T15:36:44.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T15:36:44.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T15:36:44.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:36:44.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T15:36:44.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T15:36:44.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T15:36:44.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T15:36:44.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T15:36:44.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T15:36:44.634 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T15:36:44.634 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T15:36:44.634 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T15:36:44.634 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:36:44.634 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T15:36:44.634 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T15:36:44.634 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T15:36:44.634 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T15:36:44.634 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T15:36:44.634 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T15:36:44.634 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T15:36:44.634 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:36:44.634 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:36:44.634 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:36:44.635 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:36:44.635 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:36:44.635 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:36:44.635 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T15:36:44.635 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T15:36:44.635 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:36:44.635 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:44.635 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T15:36:44.635 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T15:36:44.635 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T15:36:44.635 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T15:36:44.635 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:36:44.635 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T15:36:44.636 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T15:36:44.636 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T15:36:44.636 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T15:36:44.636 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T15:36:44.636 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T15:36:44.636 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T15:36:44.636 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:36:44.636 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:36:44.636 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:36:44.636 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:36:44.636 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:36:44.636 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:36:44.636 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T15:36:44.636 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T15:36:44.636 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:36:44.637 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:44 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:44.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:36:44.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:44.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:44.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:36:44.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:44.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T15:36:44.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-22T15:36:44.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T15:36:44.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-22T15:36:44.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:44.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:44.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:44.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T15:36:44.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T15:36:44.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:44.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:44.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:44.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T15:36:44.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T15:36:44.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T15:36:44.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T15:36:44.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T15:36:44.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T15:36:44.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T15:36:44.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T15:36:44.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T15:36:44.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T15:36:44.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:36:44.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-22T15:36:44.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-22T15:36:44.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T15:36:44.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T15:36:44.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:36:44.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T15:36:44.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T15:36:44.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T15:36:44.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T15:36:44.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T15:36:44.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T15:36:44.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T15:36:44.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T15:36:44.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T15:36:44.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:36:44.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T15:36:44.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T15:36:44.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T15:36:44.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T15:36:44.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T15:36:44.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T15:36:44.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T15:36:44.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:36:44.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:36:44.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:36:44.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:36:44.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:36:44.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:36:44.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T15:36:44.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T15:36:44.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:36:44.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:44 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:45.622 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:36:45] "GET /metrics HTTP/1.1" 200 38027 "" "Prometheus/2.43.0" 2024-04-22T15:36:45.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all mon 2024-04-22T15:36:45.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all crash 2024-04-22T15:36:45.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all mds 2024-04-22T15:36:45.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T15:36:45.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-22T15:36:45.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T15:36:45.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all nfs 2024-04-22T15:36:45.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all nvmeof 2024-04-22T15:36:45.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all node-exporter 2024-04-22T15:36:45.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all prometheus 2024-04-22T15:36:45.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all alertmanager 2024-04-22T15:36:45.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all grafana 2024-04-22T15:36:45.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all loki 2024-04-22T15:36:45.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all promtail 2024-04-22T15:36:45.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[88162]: Upgrade: Finalizing container_image settings 2024-04-22T15:36:45.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[88162]: Upgrade: Complete! 2024-04-22T15:36:45.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:45.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:45.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:45.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:45.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:45.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:36:45.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[88162]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 887 B/s rd, 0 op/s 2024-04-22T15:36:45.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[88162]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:36:45.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:45.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:36:45.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:45.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all mon 2024-04-22T15:36:45.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all crash 2024-04-22T15:36:45.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all mds 2024-04-22T15:36:45.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T15:36:45.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-22T15:36:45.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T15:36:45.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all nfs 2024-04-22T15:36:45.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all nvmeof 2024-04-22T15:36:45.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all node-exporter 2024-04-22T15:36:45.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all prometheus 2024-04-22T15:36:45.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all alertmanager 2024-04-22T15:36:45.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all grafana 2024-04-22T15:36:45.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all loki 2024-04-22T15:36:45.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all promtail 2024-04-22T15:36:45.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[82523]: Upgrade: Finalizing container_image settings 2024-04-22T15:36:45.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[82523]: Upgrade: Complete! 2024-04-22T15:36:45.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:45.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:45.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:45.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:45.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:45.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:36:45.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[82523]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 887 B/s rd, 0 op/s 2024-04-22T15:36:45.627 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[82523]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:36:45.627 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:45.627 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:36:45.627 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:45 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:45.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:45 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all mon 2024-04-22T15:36:45.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:45 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all crash 2024-04-22T15:36:45.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:45 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all mds 2024-04-22T15:36:45.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:45 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T15:36:45.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:45 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-22T15:36:45.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:45 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T15:36:45.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:45 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all nfs 2024-04-22T15:36:45.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:45 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all nvmeof 2024-04-22T15:36:45.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:45 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all node-exporter 2024-04-22T15:36:45.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:45 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all prometheus 2024-04-22T15:36:45.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:45 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all alertmanager 2024-04-22T15:36:45.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:45 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all grafana 2024-04-22T15:36:45.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:45 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all loki 2024-04-22T15:36:45.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:45 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all promtail 2024-04-22T15:36:45.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:45 smithi172 ceph-mon[65307]: Upgrade: Finalizing container_image settings 2024-04-22T15:36:45.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:45 smithi172 ceph-mon[65307]: Upgrade: Complete! 2024-04-22T15:36:45.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:45 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:45.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:45 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:45.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:45 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:45.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:45 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:45.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:45 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:45.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:45 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:36:45.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:45 smithi172 ceph-mon[65307]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 887 B/s rd, 0 op/s 2024-04-22T15:36:45.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:45 smithi172 ceph-mon[65307]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:36:45.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:45 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:45.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:45 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:36:45.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:45 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:46.338 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:36:46.227+0000 7f17b18ae700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:36:46.339 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:36:46.339 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:36:46.339 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:36:46.339 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:36:46.339 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:46.339 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:46.339 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:36:46.339 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:36:46.339 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:36:46.339 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:36:46.339 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:36:46.339 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:36:46.339 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:36:46.340 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:36:46.340 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:36:46.340 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:36:46.340 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:36:46.340 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:36:46.340 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:46.340 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:46.341 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:36:46.341 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:36:46.341 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:36:46.341 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:36:46.341 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:36:46.341 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:36:46.341 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:36:46.341 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:36:46.341 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:36:46.341 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:36:46.341 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:36:46.341 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:36:46.341 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:36:46.341 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:36:46.342 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:36:46.342 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:36:46.342 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:36:46.342 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:36:46.342 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:36:46.342 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:36:46.342 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:36:46.342 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:36:46.342 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:46.342 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:46.342 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:36:46.342 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:36:46.342 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:36:46.342 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:36:46.342 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:36:46.343 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:36:46.343 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:36:46.343 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:36:46.343 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:36:46.343 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:36:46.343 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:36:46.343 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:36:46.343 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:46.343 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:46.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-mon[88162]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:36:46.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-mon[88162]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:36:46.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:36:46.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:46.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:36:46.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:46.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-mon[82523]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:36:46.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-mon[82523]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:36:46.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:36:46.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:46.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:36:46.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:46 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:46.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:46 smithi172 ceph-mon[65307]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:36:46.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:46 smithi172 ceph-mon[65307]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:36:46.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:46 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:36:46.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:46 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:46.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:46 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:36:46.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:46 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:36:47.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:47 smithi145 ceph-mon[88162]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:36:47.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:36:47.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:36:47.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:36:47.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:36:47.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:47.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:47.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:36:47.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:36:47.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:36:47.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:36:47.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:36:47.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:36:47.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:36:47.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:36:47.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:36:47.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:36:47.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:36:47.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:36:47.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:47.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:47.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:36:47.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:36:47.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:36:47.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:36:47.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:36:47.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:36:47.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:36:47.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:36:47.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:36:47.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:36:47.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:36:47.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:36:47.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:36:47.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:36:47.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:36:47.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:36:47.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:36:47.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:36:47.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:36:47.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:36:47.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:36:47.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:36:47.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:47.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:47.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:36:47.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:36:47.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:36:47.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:36:47.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:36:47.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:36:47.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:36:47.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:36:47.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:36:47.627 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:47 smithi145 ceph-mon[82523]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:36:47.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:36:47.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:36:47.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:36:47.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:36:47.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:47.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:47.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:36:47.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:36:47.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:36:47.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:36:47.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:36:47.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:36:47.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:36:47.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:36:47.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:36:47.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:36:47.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:36:47.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:36:47.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:47.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:47.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:36:47.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:36:47.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:36:47.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:36:47.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:36:47.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:36:47.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:36:47.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:36:47.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:36:47.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:36:47.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:36:47.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:36:47.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:36:47.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:36:47.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:36:47.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:36:47.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:36:47.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:36:47.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:36:47.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:36:47.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:36:47.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:36:47.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:47.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:47.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:36:47.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:36:47.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:36:47.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:36:47.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:36:47.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:36:47.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:36:47.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:36:47.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:36:47.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:36:47.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:36:47.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:36:47.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:47.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:47.632 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:47 smithi145 ceph-mon[88162]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:36:47.632 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:47 smithi145 ceph-mon[88162]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:36:47.632 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:47 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:36:47.632 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:47 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:47.633 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:36:47.633 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:36:47.633 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:36:47.633 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:47.633 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:47.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:47 smithi145 ceph-mon[82523]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:36:47.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:47 smithi145 ceph-mon[82523]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:36:47.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:47 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:36:47.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:47 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:47 smithi172 ceph-mon[65307]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:36:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:36:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:36:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:36:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:36:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:36:47.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:36:47.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:36:47.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:36:47.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:36:47.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:36:47.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:36:47.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:36:47.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:36:47.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:36:47.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:36:47.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:36:47.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:47.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:47.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:36:47.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:36:47.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:36:47.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:36:47.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:36:47.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:36:47.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:36:47.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:36:47.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:36:47.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:36:47.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:36:47.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:36:47.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:36:47.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:36:47.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:36:47.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:36:47.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:36:47.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:36:47.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:36:47.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:36:47.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:36:47.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:36:47.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:47.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:47.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:36:47.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:36:47.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:36:47.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:36:47.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:36:47.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:36:47.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:36:47.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:36:47.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:36:47.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:36:47.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:36:47.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:36:47.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:47.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:47.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:47 smithi172 ceph-mon[65307]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:36:47.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:47 smithi172 ceph-mon[65307]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:36:47.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:47 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:36:47.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:47 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:36:48.345 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:36:48.090+0000 7f17b18ae700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:36:48.345 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:36:48.345 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:36:48.345 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:36:48.345 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:36:48.345 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:48.346 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:48.346 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:36:48.346 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:36:48.346 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:36:48.346 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:36:48.346 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:36:48.346 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:36:48.346 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:36:48.346 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:36:48.346 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:36:48.346 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:36:48.346 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:36:48.346 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:36:48.347 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:48.347 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:48.347 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:36:48.347 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:36:48.347 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:36:48.347 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:36:48.347 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:36:48.348 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:36:48.348 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:36:48.348 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:36:48.348 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:36:48.348 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:36:48.348 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:36:48.348 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:36:48.348 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:36:48.348 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:36:48.348 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:36:48.348 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:36:48.348 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:36:48.348 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:36:48.348 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:36:48.348 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:36:48.348 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:36:48.349 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:36:48.349 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:48.349 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:48.349 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:36:48.349 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:36:48.349 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:36:48.349 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:36:48.349 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:36:48.349 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:36:48.349 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:36:48.349 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:36:48.349 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:36:48.349 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:36:48.349 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:36:48.350 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:36:48.350 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:48.350 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:48.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-mon[82523]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:36:48.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-mon[82523]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:36:48.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-mon[88162]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:36:48.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:48 smithi145 ceph-mon[88162]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:36:48.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:48 smithi172 ceph-mon[65307]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:36:48.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:48 smithi172 ceph-mon[65307]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:36:49.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:49 smithi145 ceph-mon[82523]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:36:49.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:36:49.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:36:49.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:36:49.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:36:49.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:49.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:49.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:36:49.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:36:49.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:36:49.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:36:49.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:36:49.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:36:49.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:36:49.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:36:49.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:36:49.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:36:49.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:36:49.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:36:49.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:49.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:49.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:36:49.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:36:49.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:36:49.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:36:49.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:36:49.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:36:49.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:36:49.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:36:49.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:36:49.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:36:49.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:36:49.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:36:49.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:36:49.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:36:49.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:36:49.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:36:49.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:36:49.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:36:49.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:36:49.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:36:49.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:36:49.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:36:49.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:49.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:49.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:36:49.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:36:49.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:36:49.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:36:49.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:36:49.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:36:49.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:36:49.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:36:49.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:36:49.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:49 smithi145 ceph-mon[88162]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:36:49.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:36:49.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:36:49.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:36:49.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:36:49.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:49.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:49.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:36:49.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:36:49.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:36:49.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:36:49.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:36:49.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:36:49.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:36:49.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:36:49.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:36:49.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:36:49.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:36:49.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:36:49.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:49.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:49.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:36:49.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:36:49.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:36:49.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:36:49.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:36:49.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:36:49.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:36:49.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:36:49.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:36:49.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:36:49.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:36:49.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:36:49.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:36:49.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:36:49.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:36:49.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:36:49.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:36:49.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:36:49.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:36:49.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:36:49.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:36:49.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:36:49.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:49.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:49.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:36:49.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:36:49.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:36:49.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:36:49.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:36:49.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:36:49.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:36:49.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:36:49.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:36:49.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:36:49.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:36:49.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:36:49.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:49.632 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:49.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:49 smithi145 ceph-mon[82523]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 688 B/s rd, 0 op/s 2024-04-22T15:36:49.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:49 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:36:49.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:36:49.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:36:49.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:36:49.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:49.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:49.632 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:49 smithi145 ceph-mon[88162]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 688 B/s rd, 0 op/s 2024-04-22T15:36:49.632 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:49 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:36:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:49 smithi172 ceph-mon[65307]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:36:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:36:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:36:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:36:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:36:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:36:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:36:49.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:36:49.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:36:49.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:36:49.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:36:49.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:36:49.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:36:49.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:36:49.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:36:49.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:36:49.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:36:49.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:49.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:49.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:36:49.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:36:49.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:36:49.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:36:49.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:36:49.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:36:49.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:36:49.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:36:49.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:36:49.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:36:49.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:36:49.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:36:49.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:36:49.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:36:49.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:36:49.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:36:49.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:36:49.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:36:49.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:36:49.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:36:49.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:36:49.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:36:49.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:49.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:49.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:36:49.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:36:49.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:36:49.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:36:49.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:36:49.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:36:49.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:36:49.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:36:49.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:36:49.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:36:49.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:36:49.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:36:49.723 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:36:49.723 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:36:49.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:49 smithi172 ceph-mon[65307]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 688 B/s rd, 0 op/s 2024-04-22T15:36:49.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:49 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:36:50.538 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:50 smithi145 ceph-mon[88162]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:36:50.539 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:50 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:36:50.539 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:50 smithi145 ceph-mon[82523]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:36:50.539 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:50 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:36:50.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:50 smithi172 ceph-mon[65307]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:36:50.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:50 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:36:53.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:53 smithi145 ceph-mon[82523]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:36:53.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:53 smithi145 ceph-mon[88162]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:36:53.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:53 smithi172 ceph-mon[65307]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:36:55.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:55 smithi145 ceph-mon[82523]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:36:55.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:55 smithi145 ceph-mon[88162]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:36:55.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:36:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:36:55] "GET /metrics HTTP/1.1" 200 38028 "" "Prometheus/2.43.0" 2024-04-22T15:36:55.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:55 smithi172 ceph-mon[65307]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:36:57.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:57 smithi145 ceph-mon[82523]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:36:57.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:57 smithi145 ceph-mon[88162]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:36:57.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:57 smithi172 ceph-mon[65307]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:36:59.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:36:59 smithi145 ceph-mon[82523]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:36:59.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:36:59 smithi145 ceph-mon[88162]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:36:59.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:36:59 smithi172 ceph-mon[65307]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:37:01.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:01 smithi145 ceph-mon[88162]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:37:01.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:01 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:37:01.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:01 smithi145 ceph-mon[82523]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:37:01.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:01 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:37:01.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:01 smithi172 ceph-mon[65307]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:37:01.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:01 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:37:03.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:03 smithi145 ceph-mon[82523]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:37:03.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:03 smithi145 ceph-mon[88162]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:37:03.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:03 smithi172 ceph-mon[65307]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:37:04.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:04 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:37:04.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:04 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:37:04.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:04 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:37:05.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:05 smithi145 ceph-mon[82523]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:37:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:05 smithi145 ceph-mon[88162]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:37:05.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:05 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:37:05] "GET /metrics HTTP/1.1" 200 38027 "" "Prometheus/2.43.0" 2024-04-22T15:37:05.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:05 smithi172 ceph-mon[65307]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:37:07.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:07 smithi145 ceph-mon[88162]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:37:07.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:07 smithi145 ceph-mon[82523]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:37:07.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:07 smithi172 ceph-mon[65307]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:37:08.684 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch ps' 2024-04-22T15:37:09.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:09 smithi145 ceph-mon[88162]: from='client.44196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:37:09.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:09 smithi145 ceph-mon[88162]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:37:09.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:09 smithi145 ceph-mon[82523]: from='client.44196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:37:09.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:09 smithi145 ceph-mon[82523]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:37:09.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:09 smithi172 ceph-mon[65307]: from='client.44196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:37:09.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:09 smithi172 ceph-mon[65307]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:37:10.712 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:37:10.712 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (4m) 58s ago 26m 16.7M - 0.25.0 c8568f914cd2 1c58ccf37ffd 2024-04-22T15:37:10.712 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (3m) 77s ago 25m 79.5M - 9.4.12 2bacad6d85d8 2f825d15ab10 2024-04-22T15:37:10.712 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 unknown 58s ago 25m 60.5M - 3.5 e1d6a67b021e 9704631ffa0e 2024-04-22T15:37:10.712 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443,9283,8765 running (5m) 77s ago 30m 448M - 19.0.0-3223-g5883d494 3d983c26e278 e7500ef71604 2024-04-22T15:37:10.712 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (15m) 58s ago 32m 496M - 19.0.0-3223-g5883d494 3d983c26e278 b2de97566cf4 2024-04-22T15:37:10.712 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (91s) 58s ago 32m 39.4M 2048M 19.0.0-3223-g5883d494 3d983c26e278 7bfdc11120f4 2024-04-22T15:37:10.712 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (2m) 77s ago 31m 39.4M 2048M 19.0.0-3223-g5883d494 3d983c26e278 0a360b08a6d6 2024-04-22T15:37:10.712 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (64s) 58s ago 31m 19.9M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38dcfa677c18 2024-04-22T15:37:10.712 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (4m) 58s ago 27m 13.1M - 1.5.0 0da6a335fe13 8c75c26fec05 2024-04-22T15:37:10.712 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (4m) 77s ago 26m 15.9M - 1.5.0 0da6a335fe13 8fd2949e2eaf 2024-04-22T15:37:10.712 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (30m) 58s ago 30m 57.2M 2992M 17.2.0 e1d6a67b021e ea9cdc639916 2024-04-22T15:37:10.713 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (30m) 58s ago 30m 55.4M 2992M 17.2.0 e1d6a67b021e 3b9b37a8781f 2024-04-22T15:37:10.713 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (29m) 58s ago 29m 51.1M 2992M 17.2.0 e1d6a67b021e d9502dded8f8 2024-04-22T15:37:10.713 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (29m) 58s ago 29m 53.1M 2992M 17.2.0 e1d6a67b021e 8c72f30e54fc 2024-04-22T15:37:10.713 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (29m) 77s ago 29m 57.8M 3248M 17.2.0 e1d6a67b021e 80001f97ed35 2024-04-22T15:37:10.713 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (28m) 77s ago 28m 54.2M 3248M 17.2.0 e1d6a67b021e ec3ab1335b5c 2024-04-22T15:37:10.713 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (28m) 77s ago 28m 54.9M 3248M 17.2.0 e1d6a67b021e 45d475edf1ca 2024-04-22T15:37:10.713 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (27m) 77s ago 27m 53.0M 3248M 17.2.0 e1d6a67b021e cc9e1d9e7916 2024-04-22T15:37:10.713 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (4m) 77s ago 27m 63.9M - 2.43.0 a07b618ecd1d 51cf4df6e8d4 2024-04-22T15:37:10.713 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (25m) 58s ago 25m 79.2M - 17.2.0 e1d6a67b021e c90a636c947e 2024-04-22T15:37:10.713 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (25m) 77s ago 25m 76.5M - 17.2.0 e1d6a67b021e 20ac46e9bfda 2024-04-22T15:37:11.340 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 1'"'"'' 2024-04-22T15:37:11.584 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:11 smithi145 ceph-mon[88162]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:37:11.585 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:11 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:37:11.585 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:11 smithi145 ceph-mon[82523]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:37:11.585 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:11 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:37:11.670 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:11 smithi172 ceph-mon[65307]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:37:11.671 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:11 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:37:12.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:12 smithi145 ceph-mon[82523]: from='client.44202 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:37:12.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:12 smithi145 ceph-mon[88162]: from='client.44202 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:37:12.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:12 smithi172 ceph-mon[65307]: from='client.44202 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:37:13.495 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:37:13.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:13 smithi145 ceph-mon[82523]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:37:13.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:13 smithi145 ceph-mon[88162]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:37:13.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:13 smithi172 ceph-mon[65307]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:37:14.181 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph versions | jq -e '"'"'.mon | keys'"'"' | grep $sha1' 2024-04-22T15:37:14.493 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:14 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/4022030627' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:37:14.494 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:14 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/4022030627' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:37:14.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:14 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/4022030627' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:37:15.610 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:15 smithi145 ceph-mon[82523]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:37:15.610 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:15 smithi145 ceph-mon[88162]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:37:15.610 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:37:15] "GET /metrics HTTP/1.1" 200 38027 "" "Prometheus/2.43.0" 2024-04-22T15:37:15.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:15 smithi172 ceph-mon[65307]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:37:16.277 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)" 2024-04-22T15:37:16.588 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:16 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/1172028736' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:37:16.589 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:16 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/1172028736' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:37:16.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:16 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/1172028736' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:37:16.905 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 5'"'"'' 2024-04-22T15:37:17.459 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:17 smithi145 ceph-mon[88162]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:37:17.459 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:17 smithi145 ceph-mon[82523]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:37:17.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:17 smithi172 ceph-mon[65307]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:37:18.443 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:18 smithi145 ceph-mon[88162]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:37:18.444 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:18 smithi145 ceph-mon[82523]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:37:18.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:18 smithi172 ceph-mon[65307]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:37:19.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:19 smithi172 ceph-mon[65307]: from='client.54157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:37:19.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:19 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:37:19.855 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:19 smithi145 ceph-mon[88162]: from='client.54157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:37:19.855 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:19 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:37:19.855 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:19 smithi145 ceph-mon[82523]: from='client.54157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:37:19.855 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:19 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:37:20.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:20 smithi172 ceph-mon[65307]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:37:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:20 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:37:20.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:20 smithi145 ceph-mon[82523]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:37:20.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:20 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:37:20.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:20 smithi145 ceph-mon[88162]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:37:20.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:20 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:37:21.057 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:37:21.673 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch upgrade status' 2024-04-22T15:37:23.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:23 smithi145 ceph-mon[82523]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:37:23.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:23 smithi145 ceph-mon[88162]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:37:23.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:23 smithi172 ceph-mon[65307]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:37:23.720 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:37:23.720 INFO:teuthology.orchestra.run.smithi145.stdout: "target_image": null, 2024-04-22T15:37:23.720 INFO:teuthology.orchestra.run.smithi145.stdout: "in_progress": false, 2024-04-22T15:37:23.720 INFO:teuthology.orchestra.run.smithi145.stdout: "which": "", 2024-04-22T15:37:23.720 INFO:teuthology.orchestra.run.smithi145.stdout: "services_complete": [], 2024-04-22T15:37:23.720 INFO:teuthology.orchestra.run.smithi145.stdout: "progress": null, 2024-04-22T15:37:23.720 INFO:teuthology.orchestra.run.smithi145.stdout: "message": "", 2024-04-22T15:37:23.720 INFO:teuthology.orchestra.run.smithi145.stdout: "is_paused": false 2024-04-22T15:37:23.720 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:37:24.414 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph health detail' 2024-04-22T15:37:25.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:25 smithi145 ceph-mon[88162]: from='client.44226 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:37:25.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:25 smithi145 ceph-mon[88162]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:37:25.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:25 smithi145 ceph-mon[82523]: from='client.44226 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:37:25.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:25 smithi145 ceph-mon[82523]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:37:25.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:37:25] "GET /metrics HTTP/1.1" 200 38024 "" "Prometheus/2.43.0" 2024-04-22T15:37:25.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:25 smithi172 ceph-mon[65307]: from='client.44226 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:37:25.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:25 smithi172 ceph-mon[65307]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:37:26.416 INFO:teuthology.orchestra.run.smithi145.stdout:HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:37:26.416 INFO:teuthology.orchestra.run.smithi145.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:37:26.416 INFO:teuthology.orchestra.run.smithi145.stdout: daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state 2024-04-22T15:37:27.208 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2' 2024-04-22T15:37:27.515 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:27 smithi145 ceph-mon[88162]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:37:27.515 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:27 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/2078205269' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:37:27.515 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:27 smithi145 ceph-mon[82523]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:37:27.515 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:27 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/2078205269' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:37:27.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:27 smithi172 ceph-mon[65307]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:37:27.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:27 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/2078205269' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:37:29.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:29 smithi145 ceph-mon[82523]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:37:29.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:29 smithi145 ceph-mon[88162]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:37:29.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:29 smithi172 ceph-mon[65307]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:37:30.504 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:30 smithi145 ceph-mon[88162]: from='client.54175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:37:30.504 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:30 smithi145 ceph-mon[82523]: from='client.54175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:37:30.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:30 smithi172 ceph-mon[65307]: from='client.54175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:37:31.491 INFO:teuthology.orchestra.run.smithi145.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:37:31.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:31 smithi145 ceph-mon[88162]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:37:31.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:31 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:37:31.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:31 smithi145 ceph-mon[82523]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:37:31.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:31 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:37:31.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:31 smithi172 ceph-mon[65307]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:37:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:31 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:37:32.181 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-22T15:37:32.746 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:32 smithi145 ceph-mon[88162]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:37:32.746 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:32 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:32.746 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:32 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:37:32.746 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:32 smithi145 ceph-mon[88162]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:37:32.746 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:32 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:37:32.746 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:32 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:37:32.746 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:32 smithi145 ceph-mon[88162]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1014 B/s rd, 0 op/s 2024-04-22T15:37:32.746 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:32 smithi145 ceph-mon[88162]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:37:32.746 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:32 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:32.746 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:32 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:37:32.746 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:32 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:37:32.747 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:32 smithi145 ceph-mon[82523]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:37:32.747 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:32 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:32.747 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:32 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:37:32.747 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:32 smithi145 ceph-mon[82523]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:37:32.747 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:32 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:37:32.747 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:32 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:37:32.747 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:32 smithi145 ceph-mon[82523]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1014 B/s rd, 0 op/s 2024-04-22T15:37:32.747 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:32 smithi145 ceph-mon[82523]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:37:32.747 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:32 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:32.747 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:32 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:37:32.747 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:32 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:37:32.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:32 smithi172 ceph-mon[65307]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:37:32.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:32 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:32.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:32 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:37:32.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:32 smithi172 ceph-mon[65307]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:37:32.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:32 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:37:32.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:32 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:37:32.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:32 smithi172 ceph-mon[65307]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1014 B/s rd, 0 op/s 2024-04-22T15:37:32.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:32 smithi172 ceph-mon[65307]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:37:32.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:32 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:32.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:32 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:37:32.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:32 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:37:33.776 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:33 smithi145 ceph-mon[88162]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:37:33.777 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:33 smithi145 ceph-mon[88162]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:37:33.777 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:33 smithi145 ceph-mon[82523]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:37:33.777 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:33 smithi145 ceph-mon[82523]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:37:33.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:33 smithi172 ceph-mon[65307]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:37:33.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:33 smithi172 ceph-mon[65307]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:37:34.884 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:37:35.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:37:35.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-mon[88162]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 632 B/s rd, 0 op/s 2024-04-22T15:37:35.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:37:35.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-mon[82523]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 632 B/s rd, 0 op/s 2024-04-22T15:37:35.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:37:35.211+0000 7f17b18ae700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:37:35.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:37:35.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:37:35.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:37:35.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:37:35.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:37:35.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:37:35.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:37:35.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:37:35.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:37:35.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:37:35.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:37:35.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:37:35.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:37:35.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:37:35.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:37:35.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:37:35.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:37:35.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:37:35.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:37:35.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:37:35.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:37:35.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:37:35.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:37:35.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:37:35.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:37:35.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:37:35.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:37:35.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:37:35.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:37:35.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:37:35.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:37:35.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:37:35.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:37:35.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:37:35.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:37:35.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:37:35.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:37:35.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:37:35.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:37:35.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:37:35.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:37:35.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:37:35.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:37:35.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:37:35.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:37:35.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:37:35.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:37:35.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:37:35.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:37:35.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:37:35.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:37:35.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:37:35.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:37:35.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:37:35.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:37:35.378 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:37:35.378 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:37:35.378 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:37:35.378 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:37:35] "GET /metrics HTTP/1.1" 200 38025 "" "Prometheus/2.43.0" 2024-04-22T15:37:35.586 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:37:35.586 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (4m) 83s ago 26m 16.7M - 0.25.0 c8568f914cd2 1c58ccf37ffd 2024-04-22T15:37:35.586 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (4m) 101s ago 26m 79.5M - 9.4.12 2bacad6d85d8 2f825d15ab10 2024-04-22T15:37:35.586 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 unknown 83s ago 25m 60.5M - 3.5 e1d6a67b021e 9704631ffa0e 2024-04-22T15:37:35.586 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443,9283,8765 running (5m) 101s ago 31m 448M - 19.0.0-3223-g5883d494 3d983c26e278 e7500ef71604 2024-04-22T15:37:35.586 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (15m) 83s ago 32m 496M - 19.0.0-3223-g5883d494 3d983c26e278 b2de97566cf4 2024-04-22T15:37:35.586 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (116s) 83s ago 33m 39.4M 2048M 19.0.0-3223-g5883d494 3d983c26e278 7bfdc11120f4 2024-04-22T15:37:35.586 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (2m) 101s ago 31m 39.4M 2048M 19.0.0-3223-g5883d494 3d983c26e278 0a360b08a6d6 2024-04-22T15:37:35.586 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (89s) 83s ago 31m 19.9M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38dcfa677c18 2024-04-22T15:37:35.586 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (5m) 83s ago 27m 13.1M - 1.5.0 0da6a335fe13 8c75c26fec05 2024-04-22T15:37:35.587 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (5m) 101s ago 27m 15.9M - 1.5.0 0da6a335fe13 8fd2949e2eaf 2024-04-22T15:37:35.587 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (31m) 83s ago 31m 57.2M 2992M 17.2.0 e1d6a67b021e ea9cdc639916 2024-04-22T15:37:35.587 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (30m) 83s ago 30m 55.4M 2992M 17.2.0 e1d6a67b021e 3b9b37a8781f 2024-04-22T15:37:35.587 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (30m) 83s ago 30m 51.1M 2992M 17.2.0 e1d6a67b021e d9502dded8f8 2024-04-22T15:37:35.587 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (29m) 83s ago 29m 53.1M 2992M 17.2.0 e1d6a67b021e 8c72f30e54fc 2024-04-22T15:37:35.587 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (29m) 101s ago 29m 57.8M 3248M 17.2.0 e1d6a67b021e 80001f97ed35 2024-04-22T15:37:35.587 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (29m) 101s ago 29m 54.2M 3248M 17.2.0 e1d6a67b021e ec3ab1335b5c 2024-04-22T15:37:35.587 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (28m) 101s ago 28m 54.9M 3248M 17.2.0 e1d6a67b021e 45d475edf1ca 2024-04-22T15:37:35.587 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (28m) 101s ago 28m 53.0M 3248M 17.2.0 e1d6a67b021e cc9e1d9e7916 2024-04-22T15:37:35.587 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (4m) 101s ago 27m 63.9M - 2.43.0 a07b618ecd1d 51cf4df6e8d4 2024-04-22T15:37:35.587 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (25m) 83s ago 25m 79.2M - 17.2.0 e1d6a67b021e c90a636c947e 2024-04-22T15:37:35.587 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (25m) 101s ago 25m 76.5M - 17.2.0 e1d6a67b021e 20ac46e9bfda 2024-04-22T15:37:35.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:37:35.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:35 smithi172 ceph-mon[65307]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 632 B/s rd, 0 op/s 2024-04-22T15:37:36.000 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:37:36.000 INFO:teuthology.orchestra.run.smithi145.stdout: "mon": { 2024-04-22T15:37:36.000 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 3 2024-04-22T15:37:36.000 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:37:36.000 INFO:teuthology.orchestra.run.smithi145.stdout: "mgr": { 2024-04-22T15:37:36.000 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:37:36.001 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:37:36.001 INFO:teuthology.orchestra.run.smithi145.stdout: "osd": { 2024-04-22T15:37:36.001 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T15:37:36.001 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:37:36.001 INFO:teuthology.orchestra.run.smithi145.stdout: "rgw": { 2024-04-22T15:37:36.001 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:37:36.001 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:37:36.001 INFO:teuthology.orchestra.run.smithi145.stdout: "overall": { 2024-04-22T15:37:36.001 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-04-22T15:37:36.001 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 5 2024-04-22T15:37:36.001 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:37:36.001 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:37:36.352 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:37:36.352 INFO:teuthology.orchestra.run.smithi145.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", 2024-04-22T15:37:36.352 INFO:teuthology.orchestra.run.smithi145.stdout: "in_progress": true, 2024-04-22T15:37:36.353 INFO:teuthology.orchestra.run.smithi145.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (2 remaining).", 2024-04-22T15:37:36.353 INFO:teuthology.orchestra.run.smithi145.stdout: "services_complete": [], 2024-04-22T15:37:36.353 INFO:teuthology.orchestra.run.smithi145.stdout: "progress": "", 2024-04-22T15:37:36.353 INFO:teuthology.orchestra.run.smithi145.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa image", 2024-04-22T15:37:36.353 INFO:teuthology.orchestra.run.smithi145.stdout: "is_paused": false 2024-04-22T15:37:36.353 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:37:36.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:36 smithi145 ceph-mon[82523]: from='client.44244 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:37:36.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:36 smithi145 ceph-mon[82523]: from='client.54187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:37:36.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:36 smithi145 ceph-mon[82523]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:37:36.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:37:36.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:37:36.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:37:36.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:37:36.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:37:36.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:37:36.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:37:36.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:37:36.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:37:36.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:37:36.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:37:36.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:37:36.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:37:36.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:37:36.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:37:36.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:37:36.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:37:36.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:37:36.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:37:36.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:37:36.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:37:36.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:37:36.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:37:36.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:37:36.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:37:36.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:37:36.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:37:36.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:37:36.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:37:36.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:37:36.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:37:36.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:37:36.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:37:36.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:37:36.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:37:36.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:37:36.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:37:36.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:37:36.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:37:36.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:37:36.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:37:36.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:37:36.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:37:36.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:37:36.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:37:36.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:37:36.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:37:36.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:37:36.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:37:36.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:37:36.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:37:36.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:37:36.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:36 smithi145 ceph-mon[88162]: from='client.44244 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:37:36.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:36 smithi145 ceph-mon[88162]: from='client.54187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:37:36.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:36 smithi145 ceph-mon[88162]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:37:36.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:37:36.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:37:36.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:37:36.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:37:36.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:37:36.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:37:36.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:37:36.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:37:36.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:37:36.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:37:36.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:37:36.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:37:36.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:37:36.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:37:36.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:37:36.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:37:36.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:37:36.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:37:36.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:37:36.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:37:36.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:37:36.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:37:36.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:37:36.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:37:36.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:37:36.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:37:36.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:37:36.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:37:36.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:37:36.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:37:36.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:37:36.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:37:36.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:37:36.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:37:36.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:37:36.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:37:36.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:37:36.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:37:36.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:37:36.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:37:36.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:37:36.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:37:36.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:37:36.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:37:36.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:37:36.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:37:36.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:37:36.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:37:36.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:37:36.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:37:36.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:37:36.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:37:36.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:37:36.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:37:36.634 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:37:36.634 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:37:36.634 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:37:36.634 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:37:36.634 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:36 smithi145 ceph-mon[88162]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:37:36.635 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:36 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/48273027' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:37:36.635 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:37:36.635 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:37:36.635 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:37:36.635 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:37:36.635 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:37:36.636 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:37:36.636 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:36 smithi145 ceph-mon[82523]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:37:36.636 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:36 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/48273027' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:37:36.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:36 smithi172 ceph-mon[65307]: from='client.44244 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:37:36.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:36 smithi172 ceph-mon[65307]: from='client.54187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:37:36.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:36 smithi172 ceph-mon[65307]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:37:36.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:37:36.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:37:36.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:37:36.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:37:36.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:37:36.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:37:36.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:37:36.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:37:36.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:37:36.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:37:36.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:37:36.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:37:36.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:37:36.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:37:36.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:37:36.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:37:36.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:37:36.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:37:36.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:37:36.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:37:36.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:37:36.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:37:36.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:37:36.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:37:36.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:37:36.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:37:36.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:37:36.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:37:36.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:37:36.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:37:36.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:37:36.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:37:36.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:37:36.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:37:36.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:37:36.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:37:36.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:37:36.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:37:36.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:37:36.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:37:36.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:37:36.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:37:36.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:37:36.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:37:36.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:37:36.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:37:36.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:37:36.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:37:36.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:37:36.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:37:36.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:37:36.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:37:36.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:37:36.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:37:36.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:37:36.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:37:36.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:37:36.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:37:36.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:36 smithi172 ceph-mon[65307]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:37:36.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:36 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/48273027' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:37:37.354 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:37 smithi145 ceph-mon[88162]: from='client.54193 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:37:37.355 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:37 smithi145 ceph-mon[88162]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:37:37.355 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:37 smithi145 ceph-mon[82523]: from='client.54193 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:37:37.355 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:37 smithi145 ceph-mon[82523]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:37:37.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:37 smithi172 ceph-mon[65307]: from='client.54193 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:37:37.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:37 smithi172 ceph-mon[65307]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:37:38.281 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:38 smithi145 ceph-mon[82523]: from='client.44268 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:37:38.281 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:38 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:38.281 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:38 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:37:38.281 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:38 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:37:38.281 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:38 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:38.281 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:38 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:37:38.281 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:38 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:38.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:38 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:37:38.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:38 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:38.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:38 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-22T15:37:38.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:38 smithi145 ceph-mon[88162]: from='client.44268 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:37:38.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:38 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:38.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:38 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:37:38.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:38 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:37:38.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:38 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:38.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:38 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:37:38.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:38 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:38.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:38 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:37:38.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:38 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:38.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:38 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-22T15:37:38.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:38 smithi172 ceph-mon[65307]: from='client.44268 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:37:38.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:38 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:38.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:38 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:37:38.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:38 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:37:38.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:38 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:38.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:38 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:37:38.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:38 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:38.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:38 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:37:38.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:38 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:38.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:38 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-22T15:37:39.556 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:39 smithi145 ceph-mon[88162]: Upgrade: Target is version 19.0.0-3223-g5883d494 (squid) 2024-04-22T15:37:39.556 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:39 smithi145 ceph-mon[88162]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, digests ['quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb'] 2024-04-22T15:37:39.556 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:39 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all mgr 2024-04-22T15:37:39.556 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:39 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all mon 2024-04-22T15:37:39.556 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:39 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all crash 2024-04-22T15:37:39.556 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:39 smithi145 ceph-mon[88162]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-22T15:37:39.556 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:39 smithi145 ceph-mon[88162]: Upgrade: osd.0 is safe to restart 2024-04-22T15:37:39.557 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:39 smithi145 ceph-mon[88162]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 632 B/s rd, 0 op/s 2024-04-22T15:37:39.557 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:39 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:39.557 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:39 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T15:37:39.557 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:39 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:37:39.557 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:39 smithi145 ceph-mon[82523]: Upgrade: Target is version 19.0.0-3223-g5883d494 (squid) 2024-04-22T15:37:39.557 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:39 smithi145 ceph-mon[82523]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, digests ['quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb'] 2024-04-22T15:37:39.557 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:39 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all mgr 2024-04-22T15:37:39.557 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:39 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all mon 2024-04-22T15:37:39.557 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:39 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all crash 2024-04-22T15:37:39.557 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:39 smithi145 ceph-mon[82523]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-22T15:37:39.557 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:39 smithi145 ceph-mon[82523]: Upgrade: osd.0 is safe to restart 2024-04-22T15:37:39.557 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:39 smithi145 ceph-mon[82523]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 632 B/s rd, 0 op/s 2024-04-22T15:37:39.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:39 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:39.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:39 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T15:37:39.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:39 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:37:39.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:39 smithi172 ceph-mon[65307]: Upgrade: Target is version 19.0.0-3223-g5883d494 (squid) 2024-04-22T15:37:39.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:39 smithi172 ceph-mon[65307]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, digests ['quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb'] 2024-04-22T15:37:39.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:39 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all mgr 2024-04-22T15:37:39.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:39 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all mon 2024-04-22T15:37:39.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:39 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all crash 2024-04-22T15:37:39.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:39 smithi172 ceph-mon[65307]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-22T15:37:39.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:39 smithi172 ceph-mon[65307]: Upgrade: osd.0 is safe to restart 2024-04-22T15:37:39.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:39 smithi172 ceph-mon[65307]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 632 B/s rd, 0 op/s 2024-04-22T15:37:39.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:39 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:39.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:39 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T15:37:39.620 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:39 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:37:40.586 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:40 smithi145 ceph-mon[88162]: Upgrade: Updating osd.0 2024-04-22T15:37:40.586 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:40 smithi145 ceph-mon[88162]: Deploying daemon osd.0 on smithi145 2024-04-22T15:37:40.587 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:40 smithi145 ceph-mon[82523]: Upgrade: Updating osd.0 2024-04-22T15:37:40.587 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:40 smithi145 ceph-mon[82523]: Deploying daemon osd.0 on smithi145 2024-04-22T15:37:40.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:40 smithi172 ceph-mon[65307]: Upgrade: Updating osd.0 2024-04-22T15:37:40.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:40 smithi172 ceph-mon[65307]: Deploying daemon osd.0 on smithi145 2024-04-22T15:37:41.609 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:41 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:37:41.609 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:41 smithi145 ceph-mon[88162]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:37:41.610 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:41 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:37:41.610 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:41 smithi145 ceph-mon[82523]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:37:41.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:41 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:37:41.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:41 smithi172 ceph-mon[65307]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:37:42.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:42 smithi145 ceph-mon[82523]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:37:42.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:42 smithi145 ceph-mon[88162]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:37:42.623 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:42 smithi145 systemd[1]: Stopping Ceph osd.0 for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:37:42.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:42 smithi172 ceph-mon[65307]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:37:43.122 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:42 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0[29461]: 2024-04-22T15:37:42.646+0000 7f0d8b35d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:37:43.123 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:42 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0[29461]: 2024-04-22T15:37:42.646+0000 7f0d8b35d700 -1 osd.0 108 *** Got signal Terminated *** 2024-04-22T15:37:43.123 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:42 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0[29461]: 2024-04-22T15:37:42.646+0000 7f0d8b35d700 -1 osd.0 108 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T15:37:43.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:43 smithi172 ceph-mon[65307]: osd.0 marked itself down and dead 2024-04-22T15:37:43.857 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:43 smithi145 ceph-mon[88162]: osd.0 marked itself down and dead 2024-04-22T15:37:43.858 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:43 smithi145 ceph-mon[82523]: osd.0 marked itself down and dead 2024-04-22T15:37:44.122 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:43 smithi145 podman[96097]: 2024-04-22 15:37:43.853329395 +0000 UTC m=+1.329181408 container died ea9cdc639916c8f4baffb342c3099e8dab3ee4dbdf29ecd86b6f04d2a6250f88 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0, version=8, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., io.buildah.version=1.19.8, architecture=x86_64, vcs-type=git, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, io.openshift.expose-services=, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, distribution-scope=public, RELEASE=HEAD) 2024-04-22T15:37:44.501 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:44 smithi145 ceph-mon[88162]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:37:44.501 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:44 smithi145 ceph-mon[88162]: osdmap e109: 8 total, 7 up, 8 in 2024-04-22T15:37:44.501 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:44 smithi145 ceph-mon[88162]: pgmap v68: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:37:44.502 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:44 smithi145 ceph-mon[82523]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:37:44.502 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:44 smithi145 ceph-mon[82523]: osdmap e109: 8 total, 7 up, 8 in 2024-04-22T15:37:44.502 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:44 smithi145 ceph-mon[82523]: pgmap v68: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:37:44.502 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:44 smithi145 podman[96097]: 2024-04-22 15:37:44.272582606 +0000 UTC m=+1.748434611 container cleanup ea9cdc639916c8f4baffb342c3099e8dab3ee4dbdf29ecd86b6f04d2a6250f88 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0, architecture=x86_64, vcs-type=git, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, version=8, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, maintainer=Guillaume Abrioux , RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc.) 2024-04-22T15:37:44.502 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:44 smithi145 bash[96097]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0 2024-04-22T15:37:44.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:44 smithi172 ceph-mon[65307]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:37:44.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:44 smithi172 ceph-mon[65307]: osdmap e109: 8 total, 7 up, 8 in 2024-04-22T15:37:44.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:44 smithi172 ceph-mon[65307]: pgmap v68: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:37:44.802 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:44 smithi145 podman[96115]: 2024-04-22 15:37:44.499067354 +0000 UTC m=+0.638901838 container remove ea9cdc639916c8f4baffb342c3099e8dab3ee4dbdf29ecd86b6f04d2a6250f88 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, release=754, com.redhat.component=centos-stream-container, ceph=True, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, RELEASE=HEAD, architecture=x86_64, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-22T15:37:45.072 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:44 smithi145 podman[96208]: 2024-04-22 15:37:44.800101567 +0000 UTC m=+0.114036092 container create 7efd03f7e2339040259fccb7c35053ce6fb7247735edb60414cd0a3466951a9f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, ceph=True) 2024-04-22T15:37:45.073 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:44 smithi145 podman[96208]: 2024-04-22 15:37:44.704056171 +0000 UTC m=+0.017990650 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:37:45.073 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:44 smithi145 podman[96208]: 2024-04-22 15:37:44.994661453 +0000 UTC m=+0.308595945 container init 7efd03f7e2339040259fccb7c35053ce6fb7247735edb60414cd0a3466951a9f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0-deactivate, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T15:37:45.073 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:44 smithi145 podman[96208]: 2024-04-22 15:37:44.998113794 +0000 UTC m=+0.312048275 container start 7efd03f7e2339040259fccb7c35053ce6fb7247735edb60414cd0a3466951a9f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True) 2024-04-22T15:37:45.073 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:45 smithi145 podman[96208]: 2024-04-22 15:37:45.069189517 +0000 UTC m=+0.383124027 container attach 7efd03f7e2339040259fccb7c35053ce6fb7247735edb60414cd0a3466951a9f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T15:37:45.326 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:37:45] "GET /metrics HTTP/1.1" 200 38025 "" "Prometheus/2.43.0" 2024-04-22T15:37:45.326 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:45 smithi145 podman[96208]: 2024-04-22 15:37:45.305626135 +0000 UTC m=+0.619560621 container died 7efd03f7e2339040259fccb7c35053ce6fb7247735edb60414cd0a3466951a9f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0-deactivate, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-04-22T15:37:45.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:45 smithi145 ceph-mon[82523]: osdmap e110: 8 total, 7 up, 8 in 2024-04-22T15:37:45.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:45 smithi145 ceph-mon[88162]: osdmap e110: 8 total, 7 up, 8 in 2024-04-22T15:37:45.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:45 smithi172 ceph-mon[65307]: osdmap e110: 8 total, 7 up, 8 in 2024-04-22T15:37:46.328 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:45 smithi145 podman[96235]: 2024-04-22 15:37:45.890304683 +0000 UTC m=+0.576784577 container remove 7efd03f7e2339040259fccb7c35053ce6fb7247735edb60414cd0a3466951a9f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_CLEAN=True) 2024-04-22T15:37:46.328 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:45 smithi145 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.0.service: Deactivated successfully. 2024-04-22T15:37:46.328 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:45 smithi145 systemd[1]: Stopped Ceph osd.0 for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:37:46.328 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:45 smithi145 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.0.service: Consumed 14.318s CPU time. 2024-04-22T15:37:46.328 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:46 smithi145 systemd[1]: Starting Ceph osd.0 for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:37:46.613 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:46 smithi145 ceph-mon[88162]: pgmap v70: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-04-22T15:37:46.613 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:46 smithi145 ceph-mon[82523]: pgmap v70: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-04-22T15:37:46.614 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:46 smithi145 podman[96359]: 2024-04-22 15:37:46.228663672 +0000 UTC m=+0.018567280 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:37:46.614 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:46 smithi145 podman[96359]: 2024-04-22 15:37:46.345198447 +0000 UTC m=+0.135102030 container create b60ddfdf66acdc475281882845f6be030435ecbb198a7ea691e9f35348ac988c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0-activate, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, GIT_CLEAN=True) 2024-04-22T15:37:46.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:46 smithi172 ceph-mon[65307]: pgmap v70: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-04-22T15:37:46.872 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:46 smithi145 podman[96359]: 2024-04-22 15:37:46.611107557 +0000 UTC m=+0.401011150 container init b60ddfdf66acdc475281882845f6be030435ecbb198a7ea691e9f35348ac988c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0-activate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-04-22T15:37:46.872 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:46 smithi145 podman[96359]: 2024-04-22 15:37:46.614500724 +0000 UTC m=+0.404404317 container start b60ddfdf66acdc475281882845f6be030435ecbb198a7ea691e9f35348ac988c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0-activate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-04-22T15:37:46.872 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:46 smithi145 podman[96359]: 2024-04-22 15:37:46.711194579 +0000 UTC m=+0.501098164 container attach b60ddfdf66acdc475281882845f6be030435ecbb198a7ea691e9f35348ac988c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True) 2024-04-22T15:37:47.373 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0-activate[96374]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:37:47.373 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0-activate[96374]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:37:47.374 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:47 smithi145 bash[96359]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:37:47.374 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:47 smithi145 bash[96359]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:37:47.374 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:47 smithi145 bash[96359]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-22T15:37:47.374 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:47 smithi145 bash[96359]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-22T15:37:47.374 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:47 smithi145 bash[96359]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-22T15:37:47.374 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:47 smithi145 bash[96359]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-22T15:37:47.375 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:47 smithi145 bash[96359]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-22T15:37:47.375 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:47 smithi145 bash[96359]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-22T15:37:47.375 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:47 smithi145 bash[96359]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-22T15:37:47.375 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0-activate[96374]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-22T15:37:47.375 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0-activate[96374]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-22T15:37:47.375 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0-activate[96374]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-22T15:37:47.375 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0-activate[96374]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-22T15:37:47.375 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0-activate[96374]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-22T15:37:47.376 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0-activate[96374]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-22T15:37:47.376 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:47 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0-activate[96374]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-22T15:37:47.376 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:47 smithi145 podman[96359]: 2024-04-22 15:37:47.114562716 +0000 UTC m=+0.904466313 container died b60ddfdf66acdc475281882845f6be030435ecbb198a7ea691e9f35348ac988c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0-activate, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T15:37:47.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:47 smithi172 ceph-mon[65307]: Health check failed: Degraded data redundancy: 68/627 objects degraded (10.845%), 20 pgs degraded (PG_DEGRADED) 2024-04-22T15:37:47.751 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:47 smithi145 ceph-mon[88162]: Health check failed: Degraded data redundancy: 68/627 objects degraded (10.845%), 20 pgs degraded (PG_DEGRADED) 2024-04-22T15:37:47.751 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:47 smithi145 ceph-mon[82523]: Health check failed: Degraded data redundancy: 68/627 objects degraded (10.845%), 20 pgs degraded (PG_DEGRADED) 2024-04-22T15:37:48.070 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:47 smithi145 podman[96498]: 2024-04-22 15:37:47.749336043 +0000 UTC m=+0.623914890 container remove b60ddfdf66acdc475281882845f6be030435ecbb198a7ea691e9f35348ac988c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0-activate, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.build-date=20240416, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5) 2024-04-22T15:37:48.070 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:47 smithi145 podman[96564]: 2024-04-22 15:37:47.950132255 +0000 UTC m=+0.102172056 container create e4bc06317af0791486085859755b3e54ee589a1a6a1211217bb81e9dd060aa6e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T15:37:48.071 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:47 smithi145 podman[96564]: 2024-04-22 15:37:47.866463753 +0000 UTC m=+0.018503562 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:37:48.622 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:48 smithi145 podman[96564]: 2024-04-22 15:37:48.152551131 +0000 UTC m=+0.304590931 container init e4bc06317af0791486085859755b3e54ee589a1a6a1211217bb81e9dd060aa6e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True) 2024-04-22T15:37:48.623 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:48 smithi145 podman[96564]: 2024-04-22 15:37:48.155456511 +0000 UTC m=+0.307496311 container start e4bc06317af0791486085859755b3e54ee589a1a6a1211217bb81e9dd060aa6e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0, ceph=True, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-22T15:37:48.623 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:48 smithi145 ceph-osd[96583]: -- 172.21.15.145:0/2652439389 <== mon.0 v2:172.21.15.145:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55791feaf680 con 0x55791fe79400 2024-04-22T15:37:48.623 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:48 smithi145 bash[96564]: e4bc06317af0791486085859755b3e54ee589a1a6a1211217bb81e9dd060aa6e 2024-04-22T15:37:48.623 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:48 smithi145 systemd[1]: Started Ceph osd.0 for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:37:49.122 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:48 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0[96579]: 2024-04-22T15:37:48.698+0000 7f69ce0c17c0 -1 Falling back to public interface 2024-04-22T15:37:49.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:49 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:49.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:49 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:49.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:49 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:49.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:49 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:37:49.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:49 smithi145 ceph-mon[88162]: pgmap v71: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 68/627 objects degraded (10.845%) 2024-04-22T15:37:49.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:49 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:49.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:49 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:49.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:49 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:49.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:49 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:37:49.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:49 smithi145 ceph-mon[82523]: pgmap v71: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 68/627 objects degraded (10.845%) 2024-04-22T15:37:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:49 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:49 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:49 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:49 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:37:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:49 smithi172 ceph-mon[65307]: pgmap v71: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 68/627 objects degraded (10.845%) 2024-04-22T15:37:50.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:50 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:50.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:50 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:37:50.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:50 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:50.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:50 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:37:50.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:50 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:50.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:50 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:37:51.610 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:51 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:37:51.611 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:51 smithi145 ceph-mon[88162]: pgmap v72: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-04-22T15:37:51.611 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:51 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:37:51.611 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:51 smithi145 ceph-mon[82523]: pgmap v72: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-04-22T15:37:51.611 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:51 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0[96579]: 2024-04-22T15:37:51.413+0000 7f69ce0c17c0 -1 osd.0 0 read_superblock omap replica is missing. 2024-04-22T15:37:51.611 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:51 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0[96579]: 2024-04-22T15:37:51.444+0000 7f69ce0c17c0 -1 osd.0 108 log_to_monitors true 2024-04-22T15:37:51.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:51 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:37:51.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:51 smithi172 ceph-mon[65307]: pgmap v72: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-04-22T15:37:52.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:52 smithi145 ceph-mon[82523]: from='osd.0 [v2:172.21.15.145:6802/3852719334,v1:172.21.15.145:6803/3852719334]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-22T15:37:52.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:52 smithi145 ceph-mon[88162]: from='osd.0 [v2:172.21.15.145:6802/3852719334,v1:172.21.15.145:6803/3852719334]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-22T15:37:52.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:52 smithi172 ceph-mon[65307]: from='osd.0 [v2:172.21.15.145:6802/3852719334,v1:172.21.15.145:6803/3852719334]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-22T15:37:53.122 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:37:52 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0[96579]: 2024-04-22T15:37:52.806+0000 7f69bb511700 -1 osd.0 108 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T15:37:53.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:53 smithi145 ceph-mon[82523]: from='osd.0 [v2:172.21.15.145:6802/3852719334,v1:172.21.15.145:6803/3852719334]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-22T15:37:53.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:53 smithi145 ceph-mon[82523]: osdmap e111: 8 total, 7 up, 8 in 2024-04-22T15:37:53.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:53 smithi145 ceph-mon[82523]: from='osd.0 [v2:172.21.15.145:6802/3852719334,v1:172.21.15.145:6803/3852719334]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:37:53.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:53 smithi145 ceph-mon[82523]: pgmap v74: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-04-22T15:37:53.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:53 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:53.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:53 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:53.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:53 smithi145 ceph-mon[82523]: from='osd.0 [v2:172.21.15.145:6802/3852719334,v1:172.21.15.145:6803/3852719334]' entity='osd.0' 2024-04-22T15:37:53.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:53 smithi145 ceph-mon[88162]: from='osd.0 [v2:172.21.15.145:6802/3852719334,v1:172.21.15.145:6803/3852719334]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-22T15:37:53.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:53 smithi145 ceph-mon[88162]: osdmap e111: 8 total, 7 up, 8 in 2024-04-22T15:37:53.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:53 smithi145 ceph-mon[88162]: from='osd.0 [v2:172.21.15.145:6802/3852719334,v1:172.21.15.145:6803/3852719334]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:37:53.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:53 smithi145 ceph-mon[88162]: pgmap v74: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-04-22T15:37:53.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:53 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:53.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:53 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:53.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:53 smithi145 ceph-mon[88162]: from='osd.0 [v2:172.21.15.145:6802/3852719334,v1:172.21.15.145:6803/3852719334]' entity='osd.0' 2024-04-22T15:37:53.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:53 smithi172 ceph-mon[65307]: from='osd.0 [v2:172.21.15.145:6802/3852719334,v1:172.21.15.145:6803/3852719334]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-22T15:37:53.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:53 smithi172 ceph-mon[65307]: osdmap e111: 8 total, 7 up, 8 in 2024-04-22T15:37:53.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:53 smithi172 ceph-mon[65307]: from='osd.0 [v2:172.21.15.145:6802/3852719334,v1:172.21.15.145:6803/3852719334]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:37:53.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:53 smithi172 ceph-mon[65307]: pgmap v74: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-04-22T15:37:53.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:53 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:53.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:53 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:53.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:53 smithi172 ceph-mon[65307]: from='osd.0 [v2:172.21.15.145:6802/3852719334,v1:172.21.15.145:6803/3852719334]' entity='osd.0' 2024-04-22T15:37:55.122 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:54 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:55.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:54 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:55.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:54 smithi145 ceph-mon[88162]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:37:55.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:54 smithi145 ceph-mon[88162]: osd.0 [v2:172.21.15.145:6802/3852719334,v1:172.21.15.145:6803/3852719334] boot 2024-04-22T15:37:55.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:54 smithi145 ceph-mon[88162]: osdmap e112: 8 total, 8 up, 8 in 2024-04-22T15:37:55.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:54 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:37:55.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:54 smithi145 ceph-mon[88162]: pgmap v76: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-04-22T15:37:55.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:54 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:55.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:54 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:55.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:54 smithi145 ceph-mon[82523]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:37:55.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:54 smithi145 ceph-mon[82523]: osd.0 [v2:172.21.15.145:6802/3852719334,v1:172.21.15.145:6803/3852719334] boot 2024-04-22T15:37:55.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:54 smithi145 ceph-mon[82523]: osdmap e112: 8 total, 8 up, 8 in 2024-04-22T15:37:55.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:54 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:37:55.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:54 smithi145 ceph-mon[82523]: pgmap v76: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-04-22T15:37:55.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:54 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:55.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:54 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:55.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:54 smithi172 ceph-mon[65307]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:37:55.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:54 smithi172 ceph-mon[65307]: osd.0 [v2:172.21.15.145:6802/3852719334,v1:172.21.15.145:6803/3852719334] boot 2024-04-22T15:37:55.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:54 smithi172 ceph-mon[65307]: osdmap e112: 8 total, 8 up, 8 in 2024-04-22T15:37:55.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:54 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T15:37:55.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:54 smithi172 ceph-mon[65307]: pgmap v76: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-04-22T15:37:55.622 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:37:55] "GET /metrics HTTP/1.1" 200 38160 "" "Prometheus/2.43.0" 2024-04-22T15:37:55.953 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:55 smithi145 ceph-mon[82523]: osdmap e113: 8 total, 8 up, 8 in 2024-04-22T15:37:55.953 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:55 smithi145 ceph-mon[88162]: osdmap e113: 8 total, 8 up, 8 in 2024-04-22T15:37:56.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:55 smithi172 ceph-mon[65307]: osdmap e113: 8 total, 8 up, 8 in 2024-04-22T15:37:56.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:56 smithi145 ceph-mon[82523]: pgmap v78: 161 pgs: 7 active+undersized+degraded, 14 active+undersized, 140 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 28/627 objects degraded (4.466%) 2024-04-22T15:37:56.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:56 smithi145 ceph-mon[88162]: pgmap v78: 161 pgs: 7 active+undersized+degraded, 14 active+undersized, 140 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 28/627 objects degraded (4.466%) 2024-04-22T15:37:57.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:56 smithi172 ceph-mon[65307]: pgmap v78: 161 pgs: 7 active+undersized+degraded, 14 active+undersized, 140 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 28/627 objects degraded (4.466%) 2024-04-22T15:37:58.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:57 smithi145 ceph-mon[82523]: Health check update: Degraded data redundancy: 28/627 objects degraded (4.466%), 7 pgs degraded (PG_DEGRADED) 2024-04-22T15:37:58.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:57 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:58.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:57 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:58.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:57 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:37:58.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:57 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:37:58.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:57 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:58.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:57 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:37:58.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:57 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:37:58.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:57 smithi145 ceph-mon[88162]: Health check update: Degraded data redundancy: 28/627 objects degraded (4.466%), 7 pgs degraded (PG_DEGRADED) 2024-04-22T15:37:58.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:57 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:58.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:57 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:58.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:57 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:37:58.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:57 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:37:58.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:57 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:58.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:57 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:37:58.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:57 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:37:58.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:57 smithi172 ceph-mon[65307]: Health check update: Degraded data redundancy: 28/627 objects degraded (4.466%), 7 pgs degraded (PG_DEGRADED) 2024-04-22T15:37:58.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:57 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:58.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:57 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:58.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:57 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:37:58.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:57 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:37:58.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:57 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:37:58.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:57 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:37:58.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:57 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:37:59.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:58 smithi145 ceph-mon[82523]: pgmap v79: 161 pgs: 7 active+undersized+degraded, 14 active+undersized, 140 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 1008 B/s rd, 0 op/s; 28/627 objects degraded (4.466%) 2024-04-22T15:37:59.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:58 smithi145 ceph-mon[82523]: pgmap v80: 161 pgs: 7 active+undersized+degraded, 14 active+undersized, 140 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 1012 B/s rd, 0 op/s; 28/627 objects degraded (4.466%) 2024-04-22T15:37:59.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:58 smithi145 ceph-mon[82523]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:37:59.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:58 smithi145 ceph-mon[82523]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:37:59.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:58 smithi145 ceph-mon[88162]: pgmap v79: 161 pgs: 7 active+undersized+degraded, 14 active+undersized, 140 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 1008 B/s rd, 0 op/s; 28/627 objects degraded (4.466%) 2024-04-22T15:37:59.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:58 smithi145 ceph-mon[88162]: pgmap v80: 161 pgs: 7 active+undersized+degraded, 14 active+undersized, 140 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 1012 B/s rd, 0 op/s; 28/627 objects degraded (4.466%) 2024-04-22T15:37:59.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:58 smithi145 ceph-mon[88162]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:37:59.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:58 smithi145 ceph-mon[88162]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:37:59.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:58 smithi172 ceph-mon[65307]: pgmap v79: 161 pgs: 7 active+undersized+degraded, 14 active+undersized, 140 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 1008 B/s rd, 0 op/s; 28/627 objects degraded (4.466%) 2024-04-22T15:37:59.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:58 smithi172 ceph-mon[65307]: pgmap v80: 161 pgs: 7 active+undersized+degraded, 14 active+undersized, 140 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 1012 B/s rd, 0 op/s; 28/627 objects degraded (4.466%) 2024-04-22T15:37:59.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:58 smithi172 ceph-mon[65307]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:37:59.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:58 smithi172 ceph-mon[65307]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:37:59.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:37:59.255+0000 7f17b18ae700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:37:59.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:37:59.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:37:59.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:37:59.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:37:59.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:37:59.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:37:59.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:37:59.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:37:59.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:37:59.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:37:59.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:37:59.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:37:59.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:37:59.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:37:59.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:37:59.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:37:59.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:37:59.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:37:59.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:37:59.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:37:59.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:37:59.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:37:59.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:37:59.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:37:59.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:37:59.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:37:59.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:37:59.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:37:59.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:37:59.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:37:59.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:37:59.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:37:59.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:37:59.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:37:59.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:37:59.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:37:59.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:37:59.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:37:59.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:37:59.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:37:59.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:37:59.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:37:59.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:37:59.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:37:59.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:37:59.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:37:59.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:37:59.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:37:59.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:37:59.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:37:59.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:37:59.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:37:59.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:37:59.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:37:59.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:37:59.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:37:59.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:37:59.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:37:59.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:37:59.267+0000 7f17e84c2700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-04-22T15:38:00.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-mon[82523]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:38:00.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:38:00.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:38:00.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:38:00.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:38:00.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:00.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:00.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:38:00.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:38:00.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:38:00.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:38:00.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:38:00.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:38:00.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:38:00.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:38:00.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:38:00.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:38:00.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:38:00.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:38:00.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:00.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:00.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:38:00.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:38:00.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:38:00.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:38:00.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:38:00.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:38:00.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:38:00.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:38:00.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:38:00.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:38:00.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:38:00.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:38:00.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:38:00.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:38:00.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:38:00.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:38:00.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:38:00.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:38:00.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:38:00.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:38:00.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:38:00.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:38:00.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:00.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:00.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:38:00.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:38:00.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:38:00.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:38:00.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:38:00.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:38:00.127 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:38:00.127 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:38:00.127 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:38:00.127 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-mon[88162]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:38:00.127 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:38:00.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:38:00.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:38:00.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:38:00.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:00.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:00.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:38:00.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:38:00.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:38:00.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:38:00.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:38:00.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:38:00.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:38:00.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:38:00.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:38:00.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:38:00.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:38:00.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:38:00.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:00.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:00.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:38:00.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:38:00.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:38:00.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:38:00.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:38:00.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:38:00.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:38:00.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:38:00.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:38:00.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:38:00.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:38:00.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:38:00.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:38:00.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:38:00.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:38:00.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:38:00.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:38:00.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:38:00.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:38:00.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:38:00.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:38:00.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:38:00.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:00.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:00.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:38:00.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:38:00.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:38:00.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:38:00.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:38:00.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:38:00.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:38:00.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:38:00.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:38:00.131 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:38:00.131 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:38:00.131 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:38:00.132 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:00.132 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:00.132 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:38:00.132 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:00.132 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:00.132 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:00.132 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T15:38:00.132 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-mon[82523]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T15:38:00.132 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-mon[82523]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-04-22T15:38:00.132 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:38:00.132 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:38:00.132 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:38:00.132 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:00.132 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:00.133 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:38:00.133 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:00.133 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:00.133 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:00.133 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T15:38:00.133 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-mon[88162]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T15:38:00.133 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:37:59 smithi145 ceph-mon[88162]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-04-22T15:38:00.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:59 smithi172 ceph-mon[65307]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:38:00.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:38:00.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:38:00.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:38:00.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:38:00.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:00.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:00.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:38:00.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:38:00.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:38:00.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:38:00.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:38:00.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:38:00.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:38:00.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:38:00.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:38:00.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:38:00.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:38:00.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:38:00.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:00.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:00.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:38:00.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:38:00.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:38:00.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:38:00.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:38:00.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:38:00.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:38:00.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:38:00.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:38:00.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:38:00.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:38:00.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:38:00.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:38:00.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:38:00.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:38:00.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:38:00.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:38:00.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:38:00.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:38:00.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:38:00.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:38:00.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:38:00.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:00.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:00.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:38:00.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:38:00.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:38:00.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:38:00.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:38:00.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:38:00.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:38:00.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:38:00.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:38:00.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:38:00.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:38:00.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:38:00.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:00.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:00.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:59 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:38:00.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:59 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:00.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:59 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:00.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:59 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:00.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:59 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T15:38:00.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:59 smithi172 ceph-mon[65307]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T15:38:00.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:37:59 smithi172 ceph-mon[65307]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-04-22T15:38:01.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:00 smithi145 ceph-mon[82523]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 913 B/s rd, 0 op/s 2024-04-22T15:38:01.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:00 smithi145 ceph-mon[82523]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 28/627 objects degraded (4.466%), 7 pgs degraded) 2024-04-22T15:38:01.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:00 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:38:01.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:00 smithi145 ceph-mon[88162]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 913 B/s rd, 0 op/s 2024-04-22T15:38:01.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:00 smithi145 ceph-mon[88162]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 28/627 objects degraded (4.466%), 7 pgs degraded) 2024-04-22T15:38:01.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:00 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:38:01.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:00 smithi172 ceph-mon[65307]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 913 B/s rd, 0 op/s 2024-04-22T15:38:01.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:00 smithi172 ceph-mon[65307]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 28/627 objects degraded (4.466%), 7 pgs degraded) 2024-04-22T15:38:01.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:00 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:38:03.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:02 smithi145 ceph-mon[82523]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-22T15:38:03.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:02 smithi145 ceph-mon[88162]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-22T15:38:03.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:02 smithi172 ceph-mon[65307]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-22T15:38:05.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:05 smithi145 ceph-mon[88162]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:38:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:05 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:05 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:38:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:05 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:05.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:05 smithi145 ceph-mon[82523]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:38:05.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:05 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:05.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:05 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:38:05.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:05 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:05.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:05 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:38:05] "GET /metrics HTTP/1.1" 200 38180 "" "Prometheus/2.43.0" 2024-04-22T15:38:05.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:05 smithi172 ceph-mon[65307]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:38:05.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:05 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:05.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:05 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:38:05.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:05 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:06.714 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:38:07.369 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:38:07.369 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (4m) 14s ago 27m 27.0M - 0.25.0 c8568f914cd2 1c58ccf37ffd 2024-04-22T15:38:07.369 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (4m) 2m ago 26m 79.5M - 9.4.12 2bacad6d85d8 2f825d15ab10 2024-04-22T15:38:07.369 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 unknown 14s ago 26m 60.5M - 3.5 e1d6a67b021e 9704631ffa0e 2024-04-22T15:38:07.369 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443,9283,8765 running (6m) 2m ago 31m 448M - 19.0.0-3223-g5883d494 3d983c26e278 e7500ef71604 2024-04-22T15:38:07.369 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (16m) 14s ago 33m 505M - 19.0.0-3223-g5883d494 3d983c26e278 b2de97566cf4 2024-04-22T15:38:07.369 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (2m) 14s ago 33m 47.2M 2048M 19.0.0-3223-g5883d494 3d983c26e278 7bfdc11120f4 2024-04-22T15:38:07.369 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (3m) 2m ago 32m 39.4M 2048M 19.0.0-3223-g5883d494 3d983c26e278 0a360b08a6d6 2024-04-22T15:38:07.369 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (2m) 14s ago 32m 36.3M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38dcfa677c18 2024-04-22T15:38:07.369 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (5m) 14s ago 27m 15.2M - 1.5.0 0da6a335fe13 8c75c26fec05 2024-04-22T15:38:07.369 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (5m) 2m ago 27m 15.9M - 1.5.0 0da6a335fe13 8fd2949e2eaf 2024-04-22T15:38:07.369 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (19s) 14s ago 31m 12.1M 2992M 19.0.0-3223-g5883d494 3d983c26e278 e4bc06317af0 2024-04-22T15:38:07.370 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (31m) 14s ago 31m 56.0M 2992M 17.2.0 e1d6a67b021e 3b9b37a8781f 2024-04-22T15:38:07.370 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (30m) 14s ago 30m 52.1M 2992M 17.2.0 e1d6a67b021e d9502dded8f8 2024-04-22T15:38:07.370 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (30m) 14s ago 30m 53.8M 2992M 17.2.0 e1d6a67b021e 8c72f30e54fc 2024-04-22T15:38:07.370 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (29m) 2m ago 29m 57.8M 3248M 17.2.0 e1d6a67b021e 80001f97ed35 2024-04-22T15:38:07.370 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (29m) 2m ago 29m 54.2M 3248M 17.2.0 e1d6a67b021e ec3ab1335b5c 2024-04-22T15:38:07.370 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (29m) 2m ago 29m 54.9M 3248M 17.2.0 e1d6a67b021e 45d475edf1ca 2024-04-22T15:38:07.370 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (28m) 2m ago 28m 53.0M 3248M 17.2.0 e1d6a67b021e cc9e1d9e7916 2024-04-22T15:38:07.370 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (5m) 2m ago 28m 63.9M - 2.43.0 a07b618ecd1d 51cf4df6e8d4 2024-04-22T15:38:07.370 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (26m) 14s ago 26m 79.6M - 17.2.0 e1d6a67b021e c90a636c947e 2024-04-22T15:38:07.370 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (26m) 2m ago 26m 76.5M - 17.2.0 e1d6a67b021e 20ac46e9bfda 2024-04-22T15:38:07.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:07 smithi145 ceph-mon[82523]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:38:07.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:07 smithi145 ceph-mon[88162]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:38:07.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:07 smithi172 ceph-mon[65307]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:38:07.766 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:38:07.767 INFO:teuthology.orchestra.run.smithi145.stdout: "mon": { 2024-04-22T15:38:07.767 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 3 2024-04-22T15:38:07.767 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:38:07.767 INFO:teuthology.orchestra.run.smithi145.stdout: "mgr": { 2024-04-22T15:38:07.767 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:38:07.767 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:38:07.767 INFO:teuthology.orchestra.run.smithi145.stdout: "osd": { 2024-04-22T15:38:07.767 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-04-22T15:38:07.767 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 1 2024-04-22T15:38:07.767 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:38:07.767 INFO:teuthology.orchestra.run.smithi145.stdout: "rgw": { 2024-04-22T15:38:07.767 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:38:07.767 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:38:07.767 INFO:teuthology.orchestra.run.smithi145.stdout: "overall": { 2024-04-22T15:38:07.768 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-04-22T15:38:07.768 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 6 2024-04-22T15:38:07.768 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:38:07.768 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:38:08.103 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:38:08.103 INFO:teuthology.orchestra.run.smithi145.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:38:08.103 INFO:teuthology.orchestra.run.smithi145.stdout: "in_progress": true, 2024-04-22T15:38:08.103 INFO:teuthology.orchestra.run.smithi145.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (1 remaining).", 2024-04-22T15:38:08.103 INFO:teuthology.orchestra.run.smithi145.stdout: "services_complete": [], 2024-04-22T15:38:08.103 INFO:teuthology.orchestra.run.smithi145.stdout: "progress": "1/8 daemons upgraded", 2024-04-22T15:38:08.103 INFO:teuthology.orchestra.run.smithi145.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T15:38:08.103 INFO:teuthology.orchestra.run.smithi145.stdout: "is_paused": false 2024-04-22T15:38:08.103 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:38:08.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:08 smithi145 ceph-mon[88162]: from='client.54217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:38:08.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:08 smithi145 ceph-mon[88162]: from='client.54223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:38:08.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:08 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/3507696369' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:08.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:08 smithi145 ceph-mon[82523]: from='client.54217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:38:08.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:08 smithi145 ceph-mon[82523]: from='client.54223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:38:08.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:08 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/3507696369' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:08.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:08 smithi172 ceph-mon[65307]: from='client.54217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:38:08.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:08 smithi172 ceph-mon[65307]: from='client.54223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:38:08.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:08 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/3507696369' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:09.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:09 smithi172 ceph-mon[65307]: from='client.44292 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:38:09.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:09 smithi172 ceph-mon[65307]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:38:09.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:09 smithi172 ceph-mon[65307]: from='client.54241 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:38:09.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:09 smithi145 ceph-mon[88162]: from='client.44292 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:38:09.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:09 smithi145 ceph-mon[88162]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:38:09.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:09 smithi145 ceph-mon[88162]: from='client.54241 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:38:09.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:09 smithi145 ceph-mon[82523]: from='client.44292 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:38:09.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:09 smithi145 ceph-mon[82523]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:38:09.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:09 smithi145 ceph-mon[82523]: from='client.54241 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:38:11.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:11 smithi145 ceph-mon[82523]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:38:11.627 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:11 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:38:11.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:11 smithi145 ceph-mon[88162]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:38:11.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:11 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:38:11.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:11 smithi172 ceph-mon[65307]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:38:11.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:11 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:38:13.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:13 smithi145 ceph-mon[82523]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:38:13.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:13 smithi145 ceph-mon[88162]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:38:13.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:13 smithi172 ceph-mon[65307]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:38:14.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:14 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T15:38:14.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:14 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T15:38:14.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:14 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T15:38:15.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:15 smithi145 ceph-mon[82523]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:38:15.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:15 smithi145 ceph-mon[82523]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T15:38:15.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:15 smithi145 ceph-mon[82523]: Upgrade: osd.1 is safe to restart 2024-04-22T15:38:15.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:38:15] "GET /metrics HTTP/1.1" 200 38180 "" "Prometheus/2.43.0" 2024-04-22T15:38:15.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:15 smithi145 ceph-mon[88162]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:38:15.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:15 smithi145 ceph-mon[88162]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T15:38:15.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:15 smithi145 ceph-mon[88162]: Upgrade: osd.1 is safe to restart 2024-04-22T15:38:15.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:15 smithi172 ceph-mon[65307]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:38:15.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:15 smithi172 ceph-mon[65307]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-22T15:38:15.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:15 smithi172 ceph-mon[65307]: Upgrade: osd.1 is safe to restart 2024-04-22T15:38:17.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:16 smithi145 ceph-mon[82523]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:38:17.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:16 smithi145 ceph-mon[82523]: Upgrade: Updating osd.1 2024-04-22T15:38:17.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:16 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:17.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:16 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T15:38:17.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:16 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:38:17.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:16 smithi145 ceph-mon[82523]: Deploying daemon osd.1 on smithi145 2024-04-22T15:38:17.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:16 smithi145 ceph-mon[88162]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:38:17.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:16 smithi145 ceph-mon[88162]: Upgrade: Updating osd.1 2024-04-22T15:38:17.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:16 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:17.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:16 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T15:38:17.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:16 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:38:17.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:16 smithi145 ceph-mon[88162]: Deploying daemon osd.1 on smithi145 2024-04-22T15:38:17.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:16 smithi172 ceph-mon[65307]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:38:17.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:16 smithi172 ceph-mon[65307]: Upgrade: Updating osd.1 2024-04-22T15:38:17.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:16 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:17.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:16 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T15:38:17.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:16 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:38:17.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:16 smithi172 ceph-mon[65307]: Deploying daemon osd.1 on smithi145 2024-04-22T15:38:19.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:18 smithi172 ceph-mon[65307]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:38:19.334 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:18 smithi145 ceph-mon[82523]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:38:19.334 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:18 smithi145 ceph-mon[88162]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:38:19.622 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:19 smithi145 systemd[1]: Stopping Ceph osd.1 for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:38:19.958 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1[32422]: 2024-04-22T15:38:19.667+0000 7f8b670b2700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:38:19.958 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1[32422]: 2024-04-22T15:38:19.667+0000 7f8b670b2700 -1 osd.1 113 *** Got signal Terminated *** 2024-04-22T15:38:19.958 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1[32422]: 2024-04-22T15:38:19.667+0000 7f8b670b2700 -1 osd.1 113 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T15:38:20.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:19 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:38:20.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:19 smithi172 ceph-mon[65307]: osd.1 marked itself down and dead 2024-04-22T15:38:20.365 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:19 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:38:20.365 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:19 smithi145 ceph-mon[88162]: osd.1 marked itself down and dead 2024-04-22T15:38:20.365 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:19 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:38:20.365 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:19 smithi145 ceph-mon[82523]: osd.1 marked itself down and dead 2024-04-22T15:38:20.622 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:20 smithi145 podman[101014]: 2024-04-22 15:38:20.375740582 +0000 UTC m=+0.843553048 container died 3b9b37a8781ffb954a57844ff41248fddfddc3b2c7b735f3aa077f9548362cd3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1, RELEASE=HEAD, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, maintainer=Guillaume Abrioux , vcs-type=git, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, version=8, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, release=754, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-22T15:38:20.958 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:20 smithi145 podman[101014]: 2024-04-22 15:38:20.766930504 +0000 UTC m=+1.234742968 container cleanup 3b9b37a8781ffb954a57844ff41248fddfddc3b2c7b735f3aa077f9548362cd3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1, vcs-type=git, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, GIT_BRANCH=HEAD, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, ceph=True, vendor=Red Hat, Inc., io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, architecture=x86_64, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-22T15:38:20.958 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:20 smithi145 bash[101014]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1 2024-04-22T15:38:21.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:20 smithi172 ceph-mon[65307]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:38:21.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:20 smithi172 ceph-mon[65307]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:38:21.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:20 smithi172 ceph-mon[65307]: osdmap e114: 8 total, 7 up, 8 in 2024-04-22T15:38:21.340 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:20 smithi145 ceph-mon[82523]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:38:21.340 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:20 smithi145 ceph-mon[82523]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:38:21.340 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:20 smithi145 ceph-mon[82523]: osdmap e114: 8 total, 7 up, 8 in 2024-04-22T15:38:21.341 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:20 smithi145 ceph-mon[88162]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:38:21.341 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:20 smithi145 ceph-mon[88162]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:38:21.341 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:20 smithi145 ceph-mon[88162]: osdmap e114: 8 total, 7 up, 8 in 2024-04-22T15:38:21.341 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:21 smithi145 podman[101049]: 2024-04-22 15:38:21.026590322 +0000 UTC m=+0.645020037 container remove 3b9b37a8781ffb954a57844ff41248fddfddc3b2c7b735f3aa077f9548362cd3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, version=8, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, distribution-scope=public, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, ceph=True, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754) 2024-04-22T15:38:21.622 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:21 smithi145 podman[101142]: 2024-04-22 15:38:21.241057538 +0000 UTC m=+0.019124176 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:38:21.623 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:21 smithi145 podman[101142]: 2024-04-22 15:38:21.352384585 +0000 UTC m=+0.130451214 container create 45bf237e23545c8f78bba59b22a27e7f1e6fe7171cabc87ed3a8f306c4ea1f9f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1-deactivate, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416) 2024-04-22T15:38:21.623 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:21 smithi145 podman[101142]: 2024-04-22 15:38:21.580303214 +0000 UTC m=+0.358369846 container init 45bf237e23545c8f78bba59b22a27e7f1e6fe7171cabc87ed3a8f306c4ea1f9f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1-deactivate, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-22T15:38:21.623 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:21 smithi145 podman[101142]: 2024-04-22 15:38:21.5836124 +0000 UTC m=+0.361679032 container start 45bf237e23545c8f78bba59b22a27e7f1e6fe7171cabc87ed3a8f306c4ea1f9f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T15:38:21.971 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:21 smithi145 podman[101142]: 2024-04-22 15:38:21.671915069 +0000 UTC m=+0.449981707 container attach 45bf237e23545c8f78bba59b22a27e7f1e6fe7171cabc87ed3a8f306c4ea1f9f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T15:38:21.971 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:21 smithi145 podman[101142]: 2024-04-22 15:38:21.869486095 +0000 UTC m=+0.647552717 container died 45bf237e23545c8f78bba59b22a27e7f1e6fe7171cabc87ed3a8f306c4ea1f9f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:38:22.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:21 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:38:22.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:21 smithi145 ceph-mon[88162]: osdmap e115: 8 total, 7 up, 8 in 2024-04-22T15:38:22.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:21 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:38:22.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:21 smithi145 ceph-mon[82523]: osdmap e115: 8 total, 7 up, 8 in 2024-04-22T15:38:22.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:21 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:38:22.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:21 smithi172 ceph-mon[65307]: osdmap e115: 8 total, 7 up, 8 in 2024-04-22T15:38:22.872 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:22 smithi145 podman[101168]: 2024-04-22 15:38:22.486208509 +0000 UTC m=+0.611603043 container remove 45bf237e23545c8f78bba59b22a27e7f1e6fe7171cabc87ed3a8f306c4ea1f9f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1-deactivate, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T15:38:22.873 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:22 smithi145 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.1.service: Deactivated successfully. 2024-04-22T15:38:22.873 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:22 smithi145 systemd[1]: Stopped Ceph osd.1 for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:38:22.873 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:22 smithi145 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.1.service: Consumed 15.074s CPU time. 2024-04-22T15:38:22.873 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:22 smithi145 systemd[1]: Starting Ceph osd.1 for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:38:23.217 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:22 smithi145 ceph-mon[82523]: pgmap v94: 161 pgs: 53 peering, 108 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:38:23.217 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:22 smithi145 ceph-mon[82523]: Health check failed: Reduced data availability: 8 pgs inactive, 12 pgs peering (PG_AVAILABILITY) 2024-04-22T15:38:23.217 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:22 smithi145 ceph-mon[88162]: pgmap v94: 161 pgs: 53 peering, 108 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:38:23.217 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:22 smithi145 ceph-mon[88162]: Health check failed: Reduced data availability: 8 pgs inactive, 12 pgs peering (PG_AVAILABILITY) 2024-04-22T15:38:23.217 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:22 smithi145 podman[101269]: 2024-04-22 15:38:22.920639976 +0000 UTC m=+0.111571324 container create 34706a4c1d01e7bdd73ce73b2610c321987413398e1ee289b3af72e71aec670c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1-activate, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux ) 2024-04-22T15:38:23.218 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:22 smithi145 podman[101269]: 2024-04-22 15:38:22.827956898 +0000 UTC m=+0.018888261 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:38:23.218 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:23 smithi145 podman[101269]: 2024-04-22 15:38:23.131969431 +0000 UTC m=+0.322900784 container init 34706a4c1d01e7bdd73ce73b2610c321987413398e1ee289b3af72e71aec670c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1-activate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T15:38:23.218 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:23 smithi145 podman[101269]: 2024-04-22 15:38:23.135407963 +0000 UTC m=+0.326339318 container start 34706a4c1d01e7bdd73ce73b2610c321987413398e1ee289b3af72e71aec670c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1-activate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-04-22T15:38:23.218 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:23 smithi145 podman[101269]: 2024-04-22 15:38:23.214608547 +0000 UTC m=+0.405539929 container attach 34706a4c1d01e7bdd73ce73b2610c321987413398e1ee289b3af72e71aec670c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True) 2024-04-22T15:38:23.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:22 smithi172 ceph-mon[65307]: pgmap v94: 161 pgs: 53 peering, 108 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:38:23.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:22 smithi172 ceph-mon[65307]: Health check failed: Reduced data availability: 8 pgs inactive, 12 pgs peering (PG_AVAILABILITY) 2024-04-22T15:38:23.873 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:23 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1-activate[101283]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:38:23.873 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:23 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1-activate[101283]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:38:23.873 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:23 smithi145 bash[101269]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:38:23.873 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:23 smithi145 bash[101269]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:38:23.873 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:23 smithi145 bash[101269]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-22T15:38:23.873 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:23 smithi145 bash[101269]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-22T15:38:23.873 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:23 smithi145 bash[101269]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-22T15:38:23.873 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:23 smithi145 bash[101269]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-22T15:38:23.873 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:23 smithi145 bash[101269]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-22T15:38:23.873 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:23 smithi145 bash[101269]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-22T15:38:23.873 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:23 smithi145 bash[101269]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-22T15:38:23.874 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:23 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1-activate[101283]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-22T15:38:23.874 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:23 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1-activate[101283]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-22T15:38:23.874 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:23 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1-activate[101283]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-22T15:38:23.874 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:23 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1-activate[101283]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-22T15:38:23.874 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:23 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1-activate[101283]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-22T15:38:23.874 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:23 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1-activate[101283]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-22T15:38:23.874 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:23 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1-activate[101283]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-22T15:38:23.874 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:23 smithi145 podman[101269]: 2024-04-22 15:38:23.638547787 +0000 UTC m=+0.829479127 container died 34706a4c1d01e7bdd73ce73b2610c321987413398e1ee289b3af72e71aec670c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1-activate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS) 2024-04-22T15:38:24.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:24 smithi145 ceph-mon[82523]: Health check failed: Degraded data redundancy: 6/627 objects degraded (0.957%), 1 pg degraded (PG_DEGRADED) 2024-04-22T15:38:24.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:24 smithi145 ceph-mon[88162]: Health check failed: Degraded data redundancy: 6/627 objects degraded (0.957%), 1 pg degraded (PG_DEGRADED) 2024-04-22T15:38:24.373 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:24 smithi145 podman[101409]: 2024-04-22 15:38:24.218678832 +0000 UTC m=+0.575054837 container remove 34706a4c1d01e7bdd73ce73b2610c321987413398e1ee289b3af72e71aec670c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-04-22T15:38:24.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:24 smithi172 ceph-mon[65307]: Health check failed: Degraded data redundancy: 6/627 objects degraded (0.957%), 1 pg degraded (PG_DEGRADED) 2024-04-22T15:38:24.685 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:24 smithi145 podman[101471]: 2024-04-22 15:38:24.333948745 +0000 UTC m=+0.017896932 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:38:24.685 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:24 smithi145 podman[101471]: 2024-04-22 15:38:24.453000313 +0000 UTC m=+0.136948489 container create d299f622f08dc5f13aa1cd57d8097cc8d408a8bb8fe5a65adde86ba21599a304 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T15:38:24.686 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:24 smithi145 podman[101471]: 2024-04-22 15:38:24.680297751 +0000 UTC m=+0.364245928 container init d299f622f08dc5f13aa1cd57d8097cc8d408a8bb8fe5a65adde86ba21599a304 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416) 2024-04-22T15:38:24.686 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:24 smithi145 podman[101471]: 2024-04-22 15:38:24.683298716 +0000 UTC m=+0.367246902 container start d299f622f08dc5f13aa1cd57d8097cc8d408a8bb8fe5a65adde86ba21599a304 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-04-22T15:38:25.025 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:24 smithi145 ceph-osd[101490]: -- 172.21.15.145:0/2077227705 <== mon.1 v2:172.21.15.145:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x555f9b665680 con 0x555f9b62f000 2024-04-22T15:38:25.025 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:24 smithi145 bash[101471]: d299f622f08dc5f13aa1cd57d8097cc8d408a8bb8fe5a65adde86ba21599a304 2024-04-22T15:38:25.025 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:24 smithi145 systemd[1]: Started Ceph osd.1 for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:38:25.289 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1[101486]: 2024-04-22T15:38:25.221+0000 7f450317d7c0 -1 Falling back to public interface 2024-04-22T15:38:25.290 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:25 smithi145 ceph-mon[88162]: pgmap v95: 161 pgs: 1 active+undersized+degraded, 53 peering, 6 active+undersized, 101 active+clean; 585 KiB data, 119 MiB used, 715 GiB / 715 GiB avail; 6/627 objects degraded (0.957%) 2024-04-22T15:38:25.290 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:25.290 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:25.290 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:25.290 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:38:25.291 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:25 smithi145 ceph-mon[82523]: pgmap v95: 161 pgs: 1 active+undersized+degraded, 53 peering, 6 active+undersized, 101 active+clean; 585 KiB data, 119 MiB used, 715 GiB / 715 GiB avail; 6/627 objects degraded (0.957%) 2024-04-22T15:38:25.291 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:25.291 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:25.291 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:25.291 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:38:25.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:25 smithi172 ceph-mon[65307]: pgmap v95: 161 pgs: 1 active+undersized+degraded, 53 peering, 6 active+undersized, 101 active+clean; 585 KiB data, 119 MiB used, 715 GiB / 715 GiB avail; 6/627 objects degraded (0.957%) 2024-04-22T15:38:25.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:25.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:25.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:25.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:38:25.622 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:38:25] "GET /metrics HTTP/1.1" 200 38180 "" "Prometheus/2.43.0" 2024-04-22T15:38:26.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:26 smithi145 ceph-mon[88162]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 12 pgs peering) 2024-04-22T15:38:26.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:26 smithi145 ceph-mon[82523]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 12 pgs peering) 2024-04-22T15:38:26.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:26 smithi172 ceph-mon[65307]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 12 pgs peering) 2024-04-22T15:38:27.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:27 smithi145 ceph-mon[82523]: pgmap v96: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 119 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-04-22T15:38:27.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:27 smithi145 ceph-mon[88162]: pgmap v96: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 119 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-04-22T15:38:27.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:27 smithi172 ceph-mon[65307]: pgmap v96: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 119 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-04-22T15:38:28.222 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:28 smithi145 ceph-mon[88162]: from='osd.1 [v2:172.21.15.145:6810/3591166018,v1:172.21.15.145:6811/3591166018]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T15:38:28.222 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:28 smithi145 ceph-mon[88162]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T15:38:28.223 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:28 smithi145 ceph-mon[82523]: from='osd.1 [v2:172.21.15.145:6810/3591166018,v1:172.21.15.145:6811/3591166018]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T15:38:28.223 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:28 smithi145 ceph-mon[82523]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T15:38:28.223 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1[101486]: 2024-04-22T15:38:27.927+0000 7f450317d7c0 -1 osd.1 0 read_superblock omap replica is missing. 2024-04-22T15:38:28.223 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1[101486]: 2024-04-22T15:38:27.955+0000 7f450317d7c0 -1 osd.1 113 log_to_monitors true 2024-04-22T15:38:28.223 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:38:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1[101486]: 2024-04-22T15:38:28.116+0000 7f44f45d5700 -1 osd.1 113 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T15:38:28.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:28 smithi172 ceph-mon[65307]: from='osd.1 [v2:172.21.15.145:6810/3591166018,v1:172.21.15.145:6811/3591166018]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T15:38:28.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:28 smithi172 ceph-mon[65307]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T15:38:29.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:29 smithi145 ceph-mon[82523]: pgmap v97: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 119 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-04-22T15:38:29.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:29 smithi145 ceph-mon[82523]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-22T15:38:29.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:29 smithi145 ceph-mon[82523]: osdmap e116: 8 total, 7 up, 8 in 2024-04-22T15:38:29.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:29 smithi145 ceph-mon[82523]: from='osd.1 [v2:172.21.15.145:6810/3591166018,v1:172.21.15.145:6811/3591166018]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:38:29.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:29 smithi145 ceph-mon[82523]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:38:29.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:29 smithi145 ceph-mon[88162]: pgmap v97: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 119 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-04-22T15:38:29.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:29 smithi145 ceph-mon[88162]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-22T15:38:29.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:29 smithi145 ceph-mon[88162]: osdmap e116: 8 total, 7 up, 8 in 2024-04-22T15:38:29.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:29 smithi145 ceph-mon[88162]: from='osd.1 [v2:172.21.15.145:6810/3591166018,v1:172.21.15.145:6811/3591166018]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:38:29.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:29 smithi145 ceph-mon[88162]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:38:29.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:29 smithi172 ceph-mon[65307]: pgmap v97: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 119 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-04-22T15:38:29.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:29 smithi172 ceph-mon[65307]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-22T15:38:29.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:29 smithi172 ceph-mon[65307]: osdmap e116: 8 total, 7 up, 8 in 2024-04-22T15:38:29.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:29 smithi172 ceph-mon[65307]: from='osd.1 [v2:172.21.15.145:6810/3591166018,v1:172.21.15.145:6811/3591166018]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:38:29.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:29 smithi172 ceph-mon[65307]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:38:30.371 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:30 smithi145 ceph-mon[88162]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:38:30.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:30 smithi145 ceph-mon[88162]: osd.1 [v2:172.21.15.145:6810/3591166018,v1:172.21.15.145:6811/3591166018] boot 2024-04-22T15:38:30.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:30 smithi145 ceph-mon[88162]: osdmap e117: 8 total, 8 up, 8 in 2024-04-22T15:38:30.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:30 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:38:30.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:30 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:30.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:30 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:30.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:30 smithi145 ceph-mon[88162]: Health check update: Degraded data redundancy: 83/627 objects degraded (13.238%), 20 pgs degraded (PG_DEGRADED) 2024-04-22T15:38:30.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:30 smithi145 ceph-mon[88162]: osdmap e118: 8 total, 8 up, 8 in 2024-04-22T15:38:30.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:30 smithi145 ceph-mon[82523]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:38:30.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:30 smithi145 ceph-mon[82523]: osd.1 [v2:172.21.15.145:6810/3591166018,v1:172.21.15.145:6811/3591166018] boot 2024-04-22T15:38:30.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:30 smithi145 ceph-mon[82523]: osdmap e117: 8 total, 8 up, 8 in 2024-04-22T15:38:30.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:30 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:38:30.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:30 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:30.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:30 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:30.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:30 smithi145 ceph-mon[82523]: Health check update: Degraded data redundancy: 83/627 objects degraded (13.238%), 20 pgs degraded (PG_DEGRADED) 2024-04-22T15:38:30.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:30 smithi145 ceph-mon[82523]: osdmap e118: 8 total, 8 up, 8 in 2024-04-22T15:38:30.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:30 smithi172 ceph-mon[65307]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:38:30.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:30 smithi172 ceph-mon[65307]: osd.1 [v2:172.21.15.145:6810/3591166018,v1:172.21.15.145:6811/3591166018] boot 2024-04-22T15:38:30.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:30 smithi172 ceph-mon[65307]: osdmap e117: 8 total, 8 up, 8 in 2024-04-22T15:38:30.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:30 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T15:38:30.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:30 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:30.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:30 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:30.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:30 smithi172 ceph-mon[65307]: Health check update: Degraded data redundancy: 83/627 objects degraded (13.238%), 20 pgs degraded (PG_DEGRADED) 2024-04-22T15:38:30.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:30 smithi172 ceph-mon[65307]: osdmap e118: 8 total, 8 up, 8 in 2024-04-22T15:38:31.331 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:31 smithi145 ceph-mon[88162]: pgmap v100: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 83/627 objects degraded (13.238%) 2024-04-22T15:38:31.331 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:31 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:31.331 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:31 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:31.331 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:31 smithi145 ceph-mon[82523]: pgmap v100: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 83/627 objects degraded (13.238%) 2024-04-22T15:38:31.331 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:31 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:31.331 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:31 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:31.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:31 smithi172 ceph-mon[65307]: pgmap v100: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 83/627 objects degraded (13.238%) 2024-04-22T15:38:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:31 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:31 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:32.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:32 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:38:32.558 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:32 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:38:32.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:32 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:38:33.517 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:33 smithi145 ceph-mon[88162]: pgmap v102: 161 pgs: 7 active+undersized+degraded, 19 active+undersized, 135 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 36/627 objects degraded (5.742%) 2024-04-22T15:38:33.518 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:33 smithi145 ceph-mon[82523]: pgmap v102: 161 pgs: 7 active+undersized+degraded, 19 active+undersized, 135 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 36/627 objects degraded (5.742%) 2024-04-22T15:38:33.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:33 smithi172 ceph-mon[65307]: pgmap v102: 161 pgs: 7 active+undersized+degraded, 19 active+undersized, 135 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 36/627 objects degraded (5.742%) 2024-04-22T15:38:35.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:34 smithi172 ceph-mon[65307]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s 2024-04-22T15:38:35.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:35.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:35.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:38:35.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:38:35.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:34 smithi172 ceph-mon[65307]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 2 op/s 2024-04-22T15:38:35.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:34 smithi172 ceph-mon[65307]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 2.9 KiB/s rd, 2 op/s 2024-04-22T15:38:35.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:35.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:34 smithi172 ceph-mon[65307]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:38:35.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:38:35.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:38:35.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:34 smithi172 ceph-mon[65307]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:38:35.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:34 smithi172 ceph-mon[65307]: Health check update: Degraded data redundancy: 36/627 objects degraded (5.742%), 7 pgs degraded (PG_DEGRADED) 2024-04-22T15:38:35.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:34 smithi172 ceph-mon[65307]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 36/627 objects degraded (5.742%), 7 pgs degraded) 2024-04-22T15:38:35.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:38:35.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:35.291 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:34 smithi145 ceph-mon[88162]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s 2024-04-22T15:38:35.291 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:35.291 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:35.291 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:38:35.291 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:38:35.291 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:34 smithi145 ceph-mon[88162]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 2 op/s 2024-04-22T15:38:35.291 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:34 smithi145 ceph-mon[88162]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 2.9 KiB/s rd, 2 op/s 2024-04-22T15:38:35.291 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:35.291 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:34 smithi145 ceph-mon[88162]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:38:35.291 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:38:35.292 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:38:35.292 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:34 smithi145 ceph-mon[88162]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:38:35.292 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:34 smithi145 ceph-mon[88162]: Health check update: Degraded data redundancy: 36/627 objects degraded (5.742%), 7 pgs degraded (PG_DEGRADED) 2024-04-22T15:38:35.292 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:34 smithi145 ceph-mon[88162]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 36/627 objects degraded (5.742%), 7 pgs degraded) 2024-04-22T15:38:35.292 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:38:35.292 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:35.292 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:34 smithi145 ceph-mon[82523]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s 2024-04-22T15:38:35.292 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:35.292 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:35.292 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:38:35.292 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:38:35.292 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:34 smithi145 ceph-mon[82523]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 2 op/s 2024-04-22T15:38:35.292 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:34 smithi145 ceph-mon[82523]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 135 MiB used, 715 GiB / 715 GiB avail; 2.9 KiB/s rd, 2 op/s 2024-04-22T15:38:35.292 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:35.293 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:34 smithi145 ceph-mon[82523]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:38:35.293 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:38:35.293 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:38:35.293 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:34 smithi145 ceph-mon[82523]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:38:35.293 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:34 smithi145 ceph-mon[82523]: Health check update: Degraded data redundancy: 36/627 objects degraded (5.742%), 7 pgs degraded (PG_DEGRADED) 2024-04-22T15:38:35.293 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:34 smithi145 ceph-mon[82523]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 36/627 objects degraded (5.742%), 7 pgs degraded) 2024-04-22T15:38:35.293 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:38:35.293 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:35.622 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:38:35] "GET /metrics HTTP/1.1" 200 38185 "" "Prometheus/2.43.0" 2024-04-22T15:38:36.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:38:36.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:36.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:36.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:36.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:36.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:36.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:36.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:36.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:36.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:36.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T15:38:36.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T15:38:36.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:38:36.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T15:38:36.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T15:38:36.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T15:38:36.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T15:38:36.125 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T15:38:36.125 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T15:38:36.125 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T15:38:36.125 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:38:36.125 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T15:38:36.125 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T15:38:36.125 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T15:38:36.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:38:36.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:36.126 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:36.126 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:36.126 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:36.126 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:36.126 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:36.126 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:36.126 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:36.126 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:36.126 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T15:38:36.126 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T15:38:36.126 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:38:36.127 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T15:38:36.127 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T15:38:36.127 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T15:38:36.127 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T15:38:36.127 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T15:38:36.127 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T15:38:36.127 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T15:38:36.127 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:38:36.127 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T15:38:36.127 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T15:38:36.127 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T15:38:36.128 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T15:38:36.128 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T15:38:36.128 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:38:36.128 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:38:36.128 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:38:36.128 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:38:36.128 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:38:36.128 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:38:36.128 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T15:38:36.128 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T15:38:36.128 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:38:36.129 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:38:35.860+0000 7f17b18ae700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:38:36.129 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:38:36.129 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:38:36.129 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:38:36.129 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:38:36.129 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:36.129 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:36.129 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:38:36.129 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:38:36.129 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:38:36.129 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:38:36.129 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:38:36.130 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:38:36.130 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:38:36.130 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:38:36.130 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:38:36.130 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:38:36.130 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:38:36.130 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:38:36.130 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T15:38:36.130 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T15:38:36.131 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:38:36.131 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:38:36.131 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:38:36.131 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:38:36.131 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:38:36.131 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:38:36.131 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T15:38:36.131 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T15:38:36.131 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:38:36.132 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:36.132 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:36.132 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:38:36.132 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:38:36.132 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:38:36.132 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:38:36.132 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:38:36.132 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:38:36.132 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:38:36.132 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:38:36.132 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:38:36.132 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:38:36.133 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:38:36.133 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:38:36.133 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:38:36.133 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:38:36.133 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:38:36.133 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:38:36.133 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:38:36.133 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:38:36.133 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:38:36.133 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:38:36.133 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:38:36.133 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:38:36.133 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:36.134 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:36.134 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:38:36.134 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:38:36.134 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:38:36.134 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:38:36.134 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:38:36.134 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:38:36.134 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:38:36.134 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:38:36.134 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:38:36.134 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:38:36.134 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:38:36.134 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:38:36.134 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:36.134 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:36.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:38:36.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:36.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:36.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:36.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:36.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:36.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:36.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:36.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:36.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:36.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T15:38:36.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T15:38:36.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:38:36.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T15:38:36.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T15:38:36.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T15:38:36.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T15:38:36.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T15:38:36.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T15:38:36.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T15:38:36.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:38:36.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T15:38:36.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T15:38:36.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T15:38:36.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T15:38:36.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T15:38:36.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:38:36.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:38:36.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:38:36.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:38:36.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:38:36.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:38:36.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T15:38:36.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T15:38:36.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:38:37.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:36 smithi172 ceph-mon[65307]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:38:37.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:38:37.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:38:37.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:38:37.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:38:37.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:37.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:37.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:38:37.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:38:37.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:38:37.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:38:37.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:38:37.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:38:37.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:38:37.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:38:37.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:38:37.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:38:37.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:38:37.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:38:37.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:37.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:37.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:38:37.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:38:37.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:38:37.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:38:37.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:38:37.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:38:37.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:38:37.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:38:37.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:38:37.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:38:37.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:38:37.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:38:37.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:38:37.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:38:37.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:38:37.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:38:37.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:38:37.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:38:37.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:38:37.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:38:37.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:38:37.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:38:37.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:37.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:37.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:38:37.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:38:37.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:38:37.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:38:37.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:38:37.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:38:37.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:38:37.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:38:37.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:38:37.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:38:37.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:38:37.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:38:37.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:37.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:37.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:36 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all nvmeof 2024-04-22T15:38:37.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:36 smithi172 ceph-mon[65307]: Upgrade: Finalizing container_image settings 2024-04-22T15:38:37.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:36 smithi172 ceph-mon[65307]: Upgrade: Complete! 2024-04-22T15:38:37.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:36 smithi172 ceph-mon[65307]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 2.9 KiB/s rd, 2 op/s 2024-04-22T15:38:37.223 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:36 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:38:37.223 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:36 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:38:37.223 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:36 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:37.223 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:36 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:38:37.223 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:36 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:38:37.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:36 smithi145 ceph-mon[88162]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:38:37.313 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:38:37.314 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:38:37.314 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:38:37.314 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:38:37.314 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:37.314 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:37.314 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:38:37.314 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:38:37.314 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:38:37.314 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:38:37.314 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:38:37.314 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:38:37.314 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:38:37.314 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:38:37.314 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:38:37.315 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:38:37.315 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:38:37.315 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:38:37.315 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:37.315 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:37.315 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:38:37.315 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:38:37.315 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:38:37.315 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:38:37.315 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:38:37.315 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:38:37.315 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:38:37.315 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:38:37.315 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:38:37.315 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:38:37.315 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:38:37.316 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:38:37.316 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:38:37.316 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:38:37.316 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:38:37.316 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:38:37.316 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:38:37.316 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:38:37.316 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:38:37.316 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:38:37.316 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:38:37.316 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:38:37.316 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:37.316 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:37.316 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:38:37.316 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:38:37.316 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:38:37.316 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:38:37.317 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:38:37.317 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:38:37.317 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:38:37.317 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:38:37.317 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:38:37.317 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:36 smithi145 ceph-mon[82523]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:38:37.317 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:38:37.317 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:38:37.317 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:38:37.317 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:38:37.317 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:37.317 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:37.318 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:38:37.318 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:38:37.318 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:38:37.318 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:38:37.318 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:38:37.318 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:38:37.318 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:38:37.318 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:38:37.318 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:38:37.318 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:38:37.318 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:38:37.318 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:38:37.318 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:37.318 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:37.318 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:38:37.319 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:38:37.319 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:38:37.319 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:38:37.319 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:38:37.319 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:38:37.319 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:38:37.319 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:38:37.319 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:38:37.319 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:38:37.319 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:38:37.319 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:38:37.319 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:38:37.319 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:38:37.319 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:38:37.319 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:38:37.320 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:38:37.320 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:38:37.320 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:38:37.320 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:38:37.320 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:38:37.320 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:38:37.320 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:37.320 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:37.320 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:38:37.320 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:38:37.320 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:38:37.320 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:38:37.320 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:38:37.320 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:38:37.320 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:38:37.321 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:38:37.321 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:38:37.321 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:38:37.321 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:38:37.321 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:38:37.321 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:37.321 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:37.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:36 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all nvmeof 2024-04-22T15:38:37.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:36 smithi145 ceph-mon[88162]: Upgrade: Finalizing container_image settings 2024-04-22T15:38:37.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:36 smithi145 ceph-mon[88162]: Upgrade: Complete! 2024-04-22T15:38:37.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:36 smithi145 ceph-mon[88162]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 2.9 KiB/s rd, 2 op/s 2024-04-22T15:38:37.322 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:36 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:38:37.322 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:36 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:38:37.322 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:36 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:37.322 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:36 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:38:37.322 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:36 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:38:37.322 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:38:37.322 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:38:37.322 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:38:37.322 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:37.322 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:37.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:36 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all nvmeof 2024-04-22T15:38:37.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:36 smithi145 ceph-mon[82523]: Upgrade: Finalizing container_image settings 2024-04-22T15:38:37.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:36 smithi145 ceph-mon[82523]: Upgrade: Complete! 2024-04-22T15:38:37.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:36 smithi145 ceph-mon[82523]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 2.9 KiB/s rd, 2 op/s 2024-04-22T15:38:37.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:36 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:38:37.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:36 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:38:37.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:36 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:37.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:36 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:38:37.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:36 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:38:38.083 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:37 smithi145 ceph-mon[88162]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 2 op/s 2024-04-22T15:38:38.084 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:37 smithi145 ceph-mon[88162]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 2.8 KiB/s rd, 2 op/s 2024-04-22T15:38:38.084 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:37 smithi145 ceph-mon[88162]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:38:38.084 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:37 smithi145 ceph-mon[88162]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:38:38.084 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:37 smithi145 ceph-mon[82523]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 2 op/s 2024-04-22T15:38:38.084 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:37 smithi145 ceph-mon[82523]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 2.8 KiB/s rd, 2 op/s 2024-04-22T15:38:38.084 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:37 smithi145 ceph-mon[82523]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:38:38.084 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:37 smithi145 ceph-mon[82523]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:38:38.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:37 smithi172 ceph-mon[65307]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 2 op/s 2024-04-22T15:38:38.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:37 smithi172 ceph-mon[65307]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 2.8 KiB/s rd, 2 op/s 2024-04-22T15:38:38.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:37 smithi172 ceph-mon[65307]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:38:38.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:37 smithi172 ceph-mon[65307]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:38:38.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:38:38.203+0000 7f17b18ae700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:38:38.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:38:38.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:38:38.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:38:38.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:38:38.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:38.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:38.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:38:38.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:38:38.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:38:38.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:38:38.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:38:38.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:38:38.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:38:38.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:38:38.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:38:38.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:38:38.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:38:38.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:38:38.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:38.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:38.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:38:38.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:38:38.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:38:38.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:38:38.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:38:38.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:38:38.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:38:38.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:38:38.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:38:38.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:38:38.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:38:38.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:38:38.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:38:38.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:38:38.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:38:38.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:38:38.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:38:38.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:38:38.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:38:38.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:38:38.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:38:38.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:38:38.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:38.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:38.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:38:38.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:38:38.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:38:38.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:38:38.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:38:38.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:38:38.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:38:38.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:38:38.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:38:38.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:38:38.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:38:38.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:38:38.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:38.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:39.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-mon[82523]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:38:39.072 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:38:39.072 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:38:39.072 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:38:39.072 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:38:39.072 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:39.072 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:39.072 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:38:39.072 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:38:39.072 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:38:39.072 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:38:39.073 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:38:39.073 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:38:39.073 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:38:39.073 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:38:39.073 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:38:39.073 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:38:39.073 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:38:39.073 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:38:39.073 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:39.073 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:39.073 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:38:39.073 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:38:39.073 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:38:39.073 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:38:39.073 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:38:39.073 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:38:39.074 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:38:39.074 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:38:39.074 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:38:39.074 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:38:39.074 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:38:39.074 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:38:39.074 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:38:39.074 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:38:39.074 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:38:39.074 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:38:39.074 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:38:39.074 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:38:39.074 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:38:39.074 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:38:39.074 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:38:39.075 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:38:39.075 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:39.075 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:39.075 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:38:39.075 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:38:39.075 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:38:39.075 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:38:39.075 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:38:39.075 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:38:39.075 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:38:39.075 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:38:39.075 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:38:39.076 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-mon[88162]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:38:39.076 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:38:39.076 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:38:39.076 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:38:39.076 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:38:39.076 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:39.076 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:39.076 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:38:39.076 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:38:39.076 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:38:39.076 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:38:39.076 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:38:39.076 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:38:39.077 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:38:39.077 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:38:39.077 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:38:39.077 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:38:39.077 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:38:39.077 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:38:39.077 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:39.077 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:39.077 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:38:39.077 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:38:39.077 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:38:39.077 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:38:39.077 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:38:39.077 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:38:39.077 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:38:39.078 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:38:39.078 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:38:39.078 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:38:39.078 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:38:39.078 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:38:39.078 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:38:39.078 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:38:39.078 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:38:39.078 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:38:39.078 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:38:39.078 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:38:39.078 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:38:39.078 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:38:39.078 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:38:39.079 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:38:39.079 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:39.079 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:39.079 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:38:39.079 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:38:39.079 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:38:39.079 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:38:39.079 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:38:39.079 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:38:39.079 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:38:39.079 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:38:39.079 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:38:39.080 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:38:39.080 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:38:39.080 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:38:39.080 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:39.080 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:39.080 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:38:39.080 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:38:39.080 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:38:39.080 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-mon[82523]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:38:39.080 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-mon[82523]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:38:39.080 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:39.080 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-mon[82523]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:38:39.081 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:38:39.081 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:38:39.081 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-mon[82523]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:38:39.081 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:38:39.081 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:38:39.081 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:38:39.081 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:39.081 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:39.081 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:38:39.081 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:38:39.081 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:38:39.081 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-mon[88162]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:38:39.082 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-mon[88162]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:38:39.082 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:39.082 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-mon[88162]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:38:39.082 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:38:39.082 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:38:39.082 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:38 smithi145 ceph-mon[88162]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:38:39.094 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch ps' 2024-04-22T15:38:39.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:38 smithi172 ceph-mon[65307]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:38:39.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:38:39.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:38:39.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:38:39.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:38:39.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:39.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:39.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:38:39.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:38:39.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:38:39.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:38:39.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:38:39.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:38:39.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:38:39.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:38:39.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:38:39.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:38:39.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:38:39.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:38:39.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:39.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:39.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:38:39.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:38:39.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:38:39.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:38:39.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:38:39.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:38:39.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:38:39.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:38:39.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:38:39.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:38:39.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:38:39.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:38:39.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:38:39.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:38:39.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:38:39.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:38:39.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:38:39.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:38:39.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:38:39.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:38:39.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:38:39.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:38:39.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:39.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:39.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:38:39.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:38:39.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:38:39.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:38:39.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:38:39.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:38:39.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:38:39.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:38:39.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:38:39.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:38:39.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:38:39.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:38:39.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:39.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:39.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:38 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:38:39.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:38 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:38:39.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:38 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:38:39.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:38 smithi172 ceph-mon[65307]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:38:39.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:38 smithi172 ceph-mon[65307]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:38:39.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:38 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:39.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:38 smithi172 ceph-mon[65307]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:38:39.223 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:38 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:38:39.223 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:38 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:38:39.223 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:38 smithi172 ceph-mon[65307]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:38:40.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:39 smithi172 ceph-mon[65307]: from='client.44316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:38:40.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:39 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:40.253 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:39 smithi145 ceph-mon[88162]: from='client.44316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:38:40.254 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:39 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:40.254 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:39 smithi145 ceph-mon[82523]: from='client.44316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:38:40.254 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:39 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:38:40.563 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:38:40.367+0000 7f17b18ae700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:38:40.563 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:38:40.563 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:38:40.563 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:38:40.563 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:38:40.564 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:40.564 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:40.564 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:38:40.564 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:38:40.564 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:38:40.564 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:38:40.564 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:38:40.564 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:38:40.564 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:38:40.564 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:38:40.564 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:38:40.564 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:38:40.564 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:38:40.564 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:38:40.565 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:40.565 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:40.565 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:38:40.565 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:38:40.565 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:38:40.565 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:38:40.565 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:38:40.565 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:38:40.566 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:38:40.566 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:38:40.566 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:38:40.566 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:38:40.566 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:38:40.566 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:38:40.566 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:38:40.566 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:38:40.566 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:38:40.566 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:38:40.566 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:38:40.566 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:38:40.566 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:38:40.567 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:38:40.567 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:38:40.567 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:38:40.567 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:40.567 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:40.567 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:38:40.567 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:38:40.567 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:38:40.567 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:38:40.567 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:38:40.567 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:38:40.567 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:38:40.567 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:38:40.567 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:38:40.567 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:38:40.568 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:38:40.568 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:38:40.568 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:40.568 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:41.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:40 smithi172 ceph-mon[65307]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 817 B/s rd, 0 op/s 2024-04-22T15:38:41.368 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-mon[88162]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 817 B/s rd, 0 op/s 2024-04-22T15:38:41.368 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:40 smithi145 ceph-mon[82523]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 817 B/s rd, 0 op/s 2024-04-22T15:38:42.101 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:38:42.102 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (5m) 12s ago 27m 27.0M - 0.25.0 c8568f914cd2 1c58ccf37ffd 2024-04-22T15:38:42.102 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (5m) 2m ago 27m 79.5M - 9.4.12 2bacad6d85d8 2f825d15ab10 2024-04-22T15:38:42.102 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 unknown 12s ago 26m 60.6M - 3.5 e1d6a67b021e 9704631ffa0e 2024-04-22T15:38:42.102 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443,9283,8765 running (6m) 2m ago 32m 448M - 19.0.0-3223-g5883d494 3d983c26e278 e7500ef71604 2024-04-22T15:38:42.102 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (16m) 12s ago 34m 506M - 19.0.0-3223-g5883d494 3d983c26e278 b2de97566cf4 2024-04-22T15:38:42.102 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (3m) 12s ago 34m 48.7M 2048M 19.0.0-3223-g5883d494 3d983c26e278 7bfdc11120f4 2024-04-22T15:38:42.102 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (3m) 2m ago 32m 39.4M 2048M 19.0.0-3223-g5883d494 3d983c26e278 0a360b08a6d6 2024-04-22T15:38:42.102 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (2m) 12s ago 32m 38.0M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38dcfa677c18 2024-04-22T15:38:42.102 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (6m) 12s ago 28m 15.2M - 1.5.0 0da6a335fe13 8c75c26fec05 2024-04-22T15:38:42.102 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (6m) 2m ago 28m 15.9M - 1.5.0 0da6a335fe13 8fd2949e2eaf 2024-04-22T15:38:42.102 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (54s) 12s ago 32m 49.7M 2992M 19.0.0-3223-g5883d494 3d983c26e278 e4bc06317af0 2024-04-22T15:38:42.103 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (17s) 12s ago 31m 12.4M 2992M 19.0.0-3223-g5883d494 3d983c26e278 d299f622f08d 2024-04-22T15:38:42.103 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (31m) 12s ago 31m 52.1M 2992M 17.2.0 e1d6a67b021e d9502dded8f8 2024-04-22T15:38:42.103 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (30m) 12s ago 30m 54.2M 2992M 17.2.0 e1d6a67b021e 8c72f30e54fc 2024-04-22T15:38:42.103 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (30m) 2m ago 30m 57.8M 3248M 17.2.0 e1d6a67b021e 80001f97ed35 2024-04-22T15:38:42.103 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (30m) 2m ago 30m 54.2M 3248M 17.2.0 e1d6a67b021e ec3ab1335b5c 2024-04-22T15:38:42.103 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (29m) 2m ago 29m 54.9M 3248M 17.2.0 e1d6a67b021e 45d475edf1ca 2024-04-22T15:38:42.103 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (29m) 2m ago 29m 53.0M 3248M 17.2.0 e1d6a67b021e cc9e1d9e7916 2024-04-22T15:38:42.103 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (5m) 2m ago 28m 63.9M - 2.43.0 a07b618ecd1d 51cf4df6e8d4 2024-04-22T15:38:42.103 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (27m) 12s ago 27m 79.7M - 17.2.0 e1d6a67b021e c90a636c947e 2024-04-22T15:38:42.103 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (27m) 2m ago 27m 76.5M - 17.2.0 e1d6a67b021e 20ac46e9bfda 2024-04-22T15:38:42.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:41 smithi172 ceph-mon[65307]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:38:42.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:38:42.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:38:42.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:38:42.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:38:42.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:42.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:42.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:38:42.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:38:42.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:38:42.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:38:42.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:38:42.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:38:42.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:38:42.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:38:42.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:38:42.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:38:42.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:38:42.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:38:42.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:42.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:42.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:38:42.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:38:42.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:38:42.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:38:42.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:38:42.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:38:42.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:38:42.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:38:42.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:38:42.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:38:42.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:38:42.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:38:42.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:38:42.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:38:42.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:38:42.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:38:42.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:38:42.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:38:42.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:38:42.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:38:42.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:38:42.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:38:42.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:42.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:42.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:38:42.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:38:42.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:38:42.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:38:42.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:38:42.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:38:42.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:38:42.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:38:42.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:38:42.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:38:42.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:38:42.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:38:42.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:42.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:42.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:41 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:38:42.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:41 smithi145 ceph-mon[88162]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:38:42.373 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:38:42.373 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:38:42.373 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:38:42.373 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:38:42.373 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:42.373 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:42.373 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:38:42.373 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:38:42.373 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:38:42.373 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:38:42.373 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:38:42.373 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:38:42.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:38:42.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:38:42.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:38:42.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:38:42.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:38:42.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:38:42.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:42.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:42.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:38:42.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:38:42.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:38:42.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:38:42.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:38:42.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:38:42.375 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:38:42.375 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:38:42.375 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:38:42.375 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:38:42.375 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:38:42.375 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:38:42.375 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:38:42.375 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:38:42.375 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:38:42.375 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:38:42.375 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:38:42.375 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:38:42.375 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:38:42.375 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:38:42.376 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:38:42.376 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:38:42.376 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:42.376 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:42.376 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:38:42.376 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:38:42.376 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:38:42.376 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:38:42.376 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:38:42.376 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:38:42.376 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:38:42.376 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:38:42.376 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:38:42.377 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:41 smithi145 ceph-mon[82523]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:38:42.377 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:38:42.377 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:38:42.377 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:38:42.377 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:38:42.377 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:42.377 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:42.377 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:38:42.377 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:38:42.377 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:38:42.377 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:38:42.378 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:38:42.378 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:38:42.378 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:38:42.378 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:38:42.378 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:38:42.378 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:38:42.378 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:38:42.378 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:38:42.378 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:42.378 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:42.378 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:38:42.378 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:38:42.378 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:38:42.378 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:38:42.379 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:38:42.379 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:38:42.379 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:38:42.379 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:38:42.379 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:38:42.379 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:38:42.379 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:38:42.379 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:38:42.379 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:38:42.379 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:38:42.379 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:38:42.379 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:38:42.379 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:38:42.380 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:38:42.380 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:38:42.380 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:38:42.380 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:38:42.380 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:38:42.380 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:42.380 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:42.380 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:38:42.380 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:38:42.380 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:38:42.380 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:38:42.380 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:38:42.380 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:38:42.380 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:38:42.381 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:38:42.381 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:38:42.381 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:38:42.381 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:38:42.381 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:38:42.381 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:42.381 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:42.381 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:41 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:38:42.381 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:38:42.381 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:38:42.381 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:38:42.382 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:38:42.382 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:38:42.382 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:41 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:38:42.858 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-04-22T15:38:43.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:42 smithi145 ceph-mon[88162]: from='client.44322 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:38:43.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:42 smithi145 ceph-mon[88162]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 817 B/s rd, 0 op/s 2024-04-22T15:38:43.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:42 smithi145 ceph-mon[82523]: from='client.44322 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:38:43.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:42 smithi145 ceph-mon[82523]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 817 B/s rd, 0 op/s 2024-04-22T15:38:43.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:42 smithi172 ceph-mon[65307]: from='client.44322 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:38:43.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:42 smithi172 ceph-mon[65307]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 817 B/s rd, 0 op/s 2024-04-22T15:38:44.917 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:38:45.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:45 smithi145 ceph-mon[82523]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 662 B/s rd, 0 op/s 2024-04-22T15:38:45.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:45 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/2447780318' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:45.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:45 smithi145 ceph-mon[88162]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 662 B/s rd, 0 op/s 2024-04-22T15:38:45.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:45 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/2447780318' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:45.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:38:45] "GET /metrics HTTP/1.1" 200 38185 "" "Prometheus/2.43.0" 2024-04-22T15:38:45.660 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 7'"'"'' 2024-04-22T15:38:45.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:45 smithi172 ceph-mon[65307]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 662 B/s rd, 0 op/s 2024-04-22T15:38:45.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:45 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/2447780318' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:38:47.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:47 smithi145 ceph-mon[88162]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:38:47.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:47 smithi145 ceph-mon[82523]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:38:47.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:47 smithi172 ceph-mon[65307]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:38:49.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:49 smithi145 ceph-mon[82523]: from='client.44334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:38:49.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:49 smithi145 ceph-mon[82523]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:38:49.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:49 smithi145 ceph-mon[88162]: from='client.44334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:38:49.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:49 smithi145 ceph-mon[88162]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:38:49.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:49 smithi172 ceph-mon[65307]: from='client.44334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:38:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:49 smithi172 ceph-mon[65307]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:38:49.993 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:38:50.536 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:50 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:38:50.537 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:50 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:38:50.543 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch upgrade status' 2024-04-22T15:38:50.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:50 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:38:51.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:51 smithi145 ceph-mon[88162]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:38:51.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:51 smithi145 ceph-mon[82523]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:38:51.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:51 smithi172 ceph-mon[65307]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:38:52.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:52 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:38:52.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:52 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:38:52.518 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:38:52.518 INFO:teuthology.orchestra.run.smithi145.stdout: "target_image": null, 2024-04-22T15:38:52.518 INFO:teuthology.orchestra.run.smithi145.stdout: "in_progress": false, 2024-04-22T15:38:52.518 INFO:teuthology.orchestra.run.smithi145.stdout: "which": "", 2024-04-22T15:38:52.518 INFO:teuthology.orchestra.run.smithi145.stdout: "services_complete": [], 2024-04-22T15:38:52.518 INFO:teuthology.orchestra.run.smithi145.stdout: "progress": null, 2024-04-22T15:38:52.518 INFO:teuthology.orchestra.run.smithi145.stdout: "message": "", 2024-04-22T15:38:52.518 INFO:teuthology.orchestra.run.smithi145.stdout: "is_paused": false 2024-04-22T15:38:52.518 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:38:52.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:52 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:38:53.139 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph health detail' 2024-04-22T15:38:53.594 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:53 smithi145 ceph-mon[88162]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:38:53.595 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:53 smithi145 ceph-mon[82523]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:38:53.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:53 smithi172 ceph-mon[65307]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:38:54.406 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:54 smithi145 ceph-mon[88162]: from='client.44340 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:38:54.406 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:54 smithi145 ceph-mon[82523]: from='client.44340 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:38:54.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:54 smithi172 ceph-mon[65307]: from='client.44340 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:38:55.180 INFO:teuthology.orchestra.run.smithi145.stdout:HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:38:55.180 INFO:teuthology.orchestra.run.smithi145.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:38:55.180 INFO:teuthology.orchestra.run.smithi145.stdout: daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state 2024-04-22T15:38:55.546 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:55 smithi145 ceph-mon[82523]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:38:55.547 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:55 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/170402886' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:38:55.547 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:38:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:38:55] "GET /metrics HTTP/1.1" 200 38186 "" "Prometheus/2.43.0" 2024-04-22T15:38:55.547 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:55 smithi145 ceph-mon[88162]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:38:55.547 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:55 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/170402886' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:38:55.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:55 smithi172 ceph-mon[65307]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:38:55.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:55 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/170402886' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:38:55.854 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1' 2024-04-22T15:38:57.454 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:57 smithi145 ceph-mon[82523]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:38:57.454 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:57 smithi145 ceph-mon[88162]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:38:57.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:57 smithi172 ceph-mon[65307]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:38:59.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:59 smithi145 ceph-mon[88162]: from='client.54289 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:38:59.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:38:59 smithi145 ceph-mon[88162]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:38:59.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:59 smithi145 ceph-mon[82523]: from='client.54289 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:38:59.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:38:59 smithi145 ceph-mon[82523]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:38:59.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:59 smithi172 ceph-mon[65307]: from='client.54289 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:38:59.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:38:59 smithi172 ceph-mon[65307]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:38:59.997 INFO:teuthology.orchestra.run.smithi145.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:39:00.700 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-22T15:39:01.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:00 smithi145 ceph-mon[82523]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:39:01.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:00 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:01.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:00 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:39:01.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:00 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:01.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:00 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:39:01.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:00 smithi145 ceph-mon[82523]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 866 B/s rd, 0 op/s 2024-04-22T15:39:01.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:00 smithi145 ceph-mon[82523]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:39:01.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:00 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:01.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:00 smithi145 ceph-mon[82523]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:39:01.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:00 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:39:01.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:00 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:01.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:00 smithi145 ceph-mon[82523]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:39:01.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:00 smithi145 ceph-mon[88162]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:39:01.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:00 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:01.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:00 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:39:01.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:00 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:01.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:00 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:39:01.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:00 smithi145 ceph-mon[88162]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 866 B/s rd, 0 op/s 2024-04-22T15:39:01.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:00 smithi145 ceph-mon[88162]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:39:01.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:00 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:01.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:00 smithi145 ceph-mon[88162]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:39:01.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:00 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:39:01.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:00 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:01.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:00 smithi145 ceph-mon[88162]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:39:01.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:00 smithi172 ceph-mon[65307]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:39:01.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:00 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:01.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:00 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:39:01.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:00 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:01.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:00 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:39:01.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:00 smithi172 ceph-mon[65307]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 866 B/s rd, 0 op/s 2024-04-22T15:39:01.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:00 smithi172 ceph-mon[65307]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:39:01.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:00 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:01.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:00 smithi172 ceph-mon[65307]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:39:01.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:00 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:39:01.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:00 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:01.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:00 smithi172 ceph-mon[65307]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:39:02.261 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:01 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:39:02.261 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:01 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:39:02.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:01 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:39:02.606 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:39:02.258+0000 7f17b18ae700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:39:02.606 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:39:02.606 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:39:02.606 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:39:02.606 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:39:02.606 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:02.606 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:02.606 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:39:02.606 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:39:02.606 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:39:02.606 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:39:02.607 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:39:02.607 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:39:02.607 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:39:02.607 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:39:02.607 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:39:02.607 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:39:02.607 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:39:02.607 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:39:02.607 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:02.607 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:02.608 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:39:02.608 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:39:02.608 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:39:02.608 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:39:02.608 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:39:02.608 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:39:02.608 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:39:02.608 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:39:02.608 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:39:02.608 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:39:02.608 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:39:02.608 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:39:02.608 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:39:02.608 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:39:02.608 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:39:02.608 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:39:02.609 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:39:02.609 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:39:02.609 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:39:02.609 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:39:02.609 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:39:02.609 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:39:02.609 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:02.609 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:02.609 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:39:02.609 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:39:02.609 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:39:02.609 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:39:02.609 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:39:02.609 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:39:02.609 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:39:02.609 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:39:02.610 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:39:02.610 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:39:02.610 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:39:02.610 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:39:02.610 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:02.610 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:02 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:03.157 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:03 smithi145 ceph-mon[82523]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:39:03.157 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:03 smithi145 ceph-mon[82523]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:39:03.157 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:39:03.157 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:39:03.157 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:39:03.157 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:39:03.157 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:03.158 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:03.158 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:39:03.158 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:39:03.158 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:39:03.158 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:39:03.158 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:39:03.158 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:39:03.158 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:39:03.158 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:39:03.158 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:39:03.158 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:39:03.158 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:39:03.158 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:39:03.158 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:03.159 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:03.159 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:39:03.159 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:39:03.159 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:39:03.159 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:39:03.159 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:39:03.159 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:39:03.159 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:39:03.159 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:39:03.159 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:39:03.159 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:39:03.159 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:39:03.159 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:39:03.159 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:39:03.160 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:39:03.160 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:39:03.160 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:39:03.160 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:39:03.160 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:39:03.160 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:39:03.160 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:39:03.160 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:39:03.160 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:39:03.160 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:03.160 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:03.160 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:39:03.160 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:39:03.160 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:39:03.160 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:39:03.161 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:39:03.161 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:39:03.161 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:39:03.161 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:39:03.161 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:39:03.161 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:03 smithi145 ceph-mon[88162]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:39:03.161 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:03 smithi145 ceph-mon[88162]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:39:03.161 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:39:03.161 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:39:03.161 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:39:03.161 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:39:03.162 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:03.162 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:03.162 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:39:03.162 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:39:03.162 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:39:03.162 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:39:03.162 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:39:03.162 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:39:03.162 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:39:03.162 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:39:03.162 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:39:03.162 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:39:03.162 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:39:03.162 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:39:03.163 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:03.163 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:03.163 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:39:03.163 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:39:03.163 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:39:03.163 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:39:03.163 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:39:03.163 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:39:03.163 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:39:03.163 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:39:03.163 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:39:03.163 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:39:03.163 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:39:03.163 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:39:03.163 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:39:03.164 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:39:03.164 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:39:03.164 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:39:03.164 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:39:03.164 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:39:03.164 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:39:03.164 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:39:03.164 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:39:03.164 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:39:03.164 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:03.164 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:03.164 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:39:03.164 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:39:03.164 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:39:03.164 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:39:03.164 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:39:03.165 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:39:03.165 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:39:03.165 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:39:03.165 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:39:03.165 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:39:03.165 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:39:03.165 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:39:03.165 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:03.165 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:03.165 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:03 smithi145 ceph-mon[82523]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:39:03.166 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:39:03.166 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:39:03.166 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:39:03.166 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:03.166 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:03.166 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:03 smithi145 ceph-mon[88162]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:39:03.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:03 smithi172 ceph-mon[65307]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:39:03.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:03 smithi172 ceph-mon[65307]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:39:03.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:39:03.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:39:03.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:39:03.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:39:03.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:03.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:03.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:39:03.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:39:03.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:39:03.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:39:03.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:39:03.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:39:03.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:39:03.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:39:03.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:39:03.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:39:03.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:39:03.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:39:03.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:03.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:03.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:39:03.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:39:03.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:39:03.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:39:03.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:39:03.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:39:03.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:39:03.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:39:03.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:39:03.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:39:03.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:39:03.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:39:03.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:39:03.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:39:03.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:39:03.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:39:03.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:39:03.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:39:03.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:39:03.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:39:03.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:39:03.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:39:03.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:03.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:03.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:39:03.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:39:03.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:39:03.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:39:03.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:39:03.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:39:03.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:39:03.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:39:03.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:39:03.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:39:03.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:39:03.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:39:03.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:03.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:03.472 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:03 smithi172 ceph-mon[65307]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:39:03.724 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:39:04.412 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:39:04.413 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (5m) 35s ago 28m 27.0M - 0.25.0 c8568f914cd2 1c58ccf37ffd 2024-04-22T15:39:04.413 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (5m) 3m ago 27m 79.5M - 9.4.12 2bacad6d85d8 2f825d15ab10 2024-04-22T15:39:04.413 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 unknown 35s ago 27m 60.6M - 3.5 e1d6a67b021e 9704631ffa0e 2024-04-22T15:39:04.413 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443,9283,8765 running (7m) 3m ago 32m 448M - 19.0.0-3223-g5883d494 3d983c26e278 e7500ef71604 2024-04-22T15:39:04.413 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (17m) 35s ago 34m 506M - 19.0.0-3223-g5883d494 3d983c26e278 b2de97566cf4 2024-04-22T15:39:04.413 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (3m) 35s ago 34m 48.7M 2048M 19.0.0-3223-g5883d494 3d983c26e278 7bfdc11120f4 2024-04-22T15:39:04.413 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (4m) 3m ago 33m 39.4M 2048M 19.0.0-3223-g5883d494 3d983c26e278 0a360b08a6d6 2024-04-22T15:39:04.413 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (2m) 35s ago 33m 38.0M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38dcfa677c18 2024-04-22T15:39:04.413 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (6m) 35s ago 28m 15.2M - 1.5.0 0da6a335fe13 8c75c26fec05 2024-04-22T15:39:04.413 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (6m) 3m ago 28m 15.9M - 1.5.0 0da6a335fe13 8fd2949e2eaf 2024-04-22T15:39:04.413 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (76s) 35s ago 32m 49.7M 2992M 19.0.0-3223-g5883d494 3d983c26e278 e4bc06317af0 2024-04-22T15:39:04.413 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (40s) 35s ago 32m 12.4M 2992M 19.0.0-3223-g5883d494 3d983c26e278 d299f622f08d 2024-04-22T15:39:04.414 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (31m) 35s ago 31m 52.1M 2992M 17.2.0 e1d6a67b021e d9502dded8f8 2024-04-22T15:39:04.414 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (31m) 35s ago 31m 54.2M 2992M 17.2.0 e1d6a67b021e 8c72f30e54fc 2024-04-22T15:39:04.414 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (30m) 3m ago 30m 57.8M 3248M 17.2.0 e1d6a67b021e 80001f97ed35 2024-04-22T15:39:04.414 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (30m) 3m ago 30m 54.2M 3248M 17.2.0 e1d6a67b021e ec3ab1335b5c 2024-04-22T15:39:04.414 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (30m) 3m ago 30m 54.9M 3248M 17.2.0 e1d6a67b021e 45d475edf1ca 2024-04-22T15:39:04.414 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (29m) 3m ago 29m 53.0M 3248M 17.2.0 e1d6a67b021e cc9e1d9e7916 2024-04-22T15:39:04.414 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (6m) 3m ago 29m 63.9M - 2.43.0 a07b618ecd1d 51cf4df6e8d4 2024-04-22T15:39:04.414 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (27m) 35s ago 27m 79.7M - 17.2.0 e1d6a67b021e c90a636c947e 2024-04-22T15:39:04.414 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (27m) 3m ago 27m 76.5M - 17.2.0 e1d6a67b021e 20ac46e9bfda 2024-04-22T15:39:04.822 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:39:04.823 INFO:teuthology.orchestra.run.smithi145.stdout: "mon": { 2024-04-22T15:39:04.823 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 3 2024-04-22T15:39:04.823 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:39:04.823 INFO:teuthology.orchestra.run.smithi145.stdout: "mgr": { 2024-04-22T15:39:04.823 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:39:04.823 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:39:04.823 INFO:teuthology.orchestra.run.smithi145.stdout: "osd": { 2024-04-22T15:39:04.823 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-04-22T15:39:04.823 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:39:04.823 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:39:04.823 INFO:teuthology.orchestra.run.smithi145.stdout: "rgw": { 2024-04-22T15:39:04.823 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:39:04.823 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:39:04.824 INFO:teuthology.orchestra.run.smithi145.stdout: "overall": { 2024-04-22T15:39:04.824 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-04-22T15:39:04.824 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 7 2024-04-22T15:39:04.824 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:39:04.824 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:39:05.173 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:39:05.173 INFO:teuthology.orchestra.run.smithi145.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:39:05.173 INFO:teuthology.orchestra.run.smithi145.stdout: "in_progress": true, 2024-04-22T15:39:05.173 INFO:teuthology.orchestra.run.smithi145.stdout: "which": "Upgrading daemons of type(s) crash,osd. Upgrade limited to 1 daemons (1 remaining).", 2024-04-22T15:39:05.173 INFO:teuthology.orchestra.run.smithi145.stdout: "services_complete": [], 2024-04-22T15:39:05.173 INFO:teuthology.orchestra.run.smithi145.stdout: "progress": "2/8 daemons upgraded", 2024-04-22T15:39:05.173 INFO:teuthology.orchestra.run.smithi145.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T15:39:05.173 INFO:teuthology.orchestra.run.smithi145.stdout: "is_paused": false 2024-04-22T15:39:05.173 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:39:05.372 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:05 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:39:05] "GET /metrics HTTP/1.1" 200 38186 "" "Prometheus/2.43.0" 2024-04-22T15:39:05.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:05 smithi145 ceph-mon[82523]: from='client.44358 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:39:05.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:05 smithi145 ceph-mon[82523]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:39:05.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:05 smithi145 ceph-mon[82523]: from='client.44364 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:39:05.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:05 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:39:05.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:05 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:05.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:05 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:39:05.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:05 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:05.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:05 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:05.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:05 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:05.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:05 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:05.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:05 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:05.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:05 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:05.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:05 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-22T15:39:05.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:05 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/3482317628' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:05.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:05 smithi145 ceph-mon[88162]: from='client.44358 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:39:05.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:05 smithi145 ceph-mon[88162]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:39:05.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:05 smithi145 ceph-mon[88162]: from='client.44364 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:39:05.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:05 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:39:05.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:05 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:05.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:05 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:39:05.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:05 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:05.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:05 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:05.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:05 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:05.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:05 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:05.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:05 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:05.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:05 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:05.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:05 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-22T15:39:05.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:05 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/3482317628' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:05 smithi172 ceph-mon[65307]: from='client.44358 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:39:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:05 smithi172 ceph-mon[65307]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:39:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:05 smithi172 ceph-mon[65307]: from='client.44364 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:39:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:05 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:39:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:05 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:05 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:39:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:05 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:05 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:05 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:05 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:05 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:05 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:05.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:05 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-22T15:39:05.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:05 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/3482317628' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:06.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:06 smithi145 ceph-mon[82523]: from='client.54307 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:39:06.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:06 smithi145 ceph-mon[82523]: Upgrade: Target is version 19.0.0-3223-g5883d494 (squid) 2024-04-22T15:39:06.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:06 smithi145 ceph-mon[82523]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, digests ['quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb'] 2024-04-22T15:39:06.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:06 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all mgr 2024-04-22T15:39:06.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:06 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all mon 2024-04-22T15:39:06.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:06 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all crash 2024-04-22T15:39:06.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:06 smithi145 ceph-mon[82523]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-22T15:39:06.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:06 smithi145 ceph-mon[82523]: Upgrade: osd.2 is safe to restart 2024-04-22T15:39:06.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:06 smithi145 ceph-mon[82523]: from='client.44382 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:39:06.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:06 smithi145 ceph-mon[88162]: from='client.54307 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:39:06.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:06 smithi145 ceph-mon[88162]: Upgrade: Target is version 19.0.0-3223-g5883d494 (squid) 2024-04-22T15:39:06.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:06 smithi145 ceph-mon[88162]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, digests ['quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb'] 2024-04-22T15:39:06.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:06 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all mgr 2024-04-22T15:39:06.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:06 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all mon 2024-04-22T15:39:06.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:06 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all crash 2024-04-22T15:39:06.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:06 smithi145 ceph-mon[88162]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-22T15:39:06.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:06 smithi145 ceph-mon[88162]: Upgrade: osd.2 is safe to restart 2024-04-22T15:39:06.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:06 smithi145 ceph-mon[88162]: from='client.44382 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:39:06.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:06 smithi172 ceph-mon[65307]: from='client.54307 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:39:06.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:06 smithi172 ceph-mon[65307]: Upgrade: Target is version 19.0.0-3223-g5883d494 (squid) 2024-04-22T15:39:06.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:06 smithi172 ceph-mon[65307]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, digests ['quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb'] 2024-04-22T15:39:06.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:06 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all mgr 2024-04-22T15:39:06.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:06 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all mon 2024-04-22T15:39:06.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:06 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all crash 2024-04-22T15:39:06.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:06 smithi172 ceph-mon[65307]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-22T15:39:06.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:06 smithi172 ceph-mon[65307]: Upgrade: osd.2 is safe to restart 2024-04-22T15:39:06.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:06 smithi172 ceph-mon[65307]: from='client.44382 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:39:07.357 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:07 smithi145 ceph-mon[88162]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:39:07.357 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:07 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:07.357 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:07 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T15:39:07.357 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:07 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:07.357 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:07 smithi145 ceph-mon[82523]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:39:07.357 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:07 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:07.357 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:07 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T15:39:07.357 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:07 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:07.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:07 smithi172 ceph-mon[65307]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:39:07.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:07 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:07.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:07 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T15:39:07.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:07 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:08.194 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:08 smithi145 ceph-mon[88162]: Upgrade: Updating osd.2 2024-04-22T15:39:08.195 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:08 smithi145 ceph-mon[88162]: Deploying daemon osd.2 on smithi145 2024-04-22T15:39:08.195 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:08 smithi145 ceph-mon[82523]: Upgrade: Updating osd.2 2024-04-22T15:39:08.195 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:08 smithi145 ceph-mon[82523]: Deploying daemon osd.2 on smithi145 2024-04-22T15:39:08.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:08 smithi172 ceph-mon[65307]: Upgrade: Updating osd.2 2024-04-22T15:39:08.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:08 smithi172 ceph-mon[65307]: Deploying daemon osd.2 on smithi145 2024-04-22T15:39:09.369 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:09 smithi172 ceph-mon[65307]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:39:09.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:09 smithi145 ceph-mon[82523]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:39:09.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:09 smithi145 ceph-mon[88162]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:39:10.108 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:09 smithi145 systemd[1]: Stopping Ceph osd.2 for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:39:10.108 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:10 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2[35352]: 2024-04-22T15:39:10.062+0000 7fc889491700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:39:10.109 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:10 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2[35352]: 2024-04-22T15:39:10.062+0000 7fc889491700 -1 osd.2 118 *** Got signal Terminated *** 2024-04-22T15:39:10.109 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:10 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2[35352]: 2024-04-22T15:39:10.062+0000 7fc889491700 -1 osd.2 118 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T15:39:10.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:10 smithi145 ceph-mon[82523]: osd.2 marked itself down and dead 2024-04-22T15:39:10.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:10 smithi145 ceph-mon[88162]: osd.2 marked itself down and dead 2024-04-22T15:39:10.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:10 smithi172 ceph-mon[65307]: osd.2 marked itself down and dead 2024-04-22T15:39:10.834 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:10 smithi145 podman[107828]: 2024-04-22 15:39:10.526104833 +0000 UTC m=+0.556783403 container died d9502dded8f8e7e5ac341e77b97e9582e0d9e4dc0528649f3adc26c17253d3e4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, architecture=x86_64, release=754, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, io.buildah.version=1.19.8, ceph=True, RELEASE=HEAD, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., vcs-type=git, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-22T15:39:11.110 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:10 smithi145 podman[107828]: 2024-04-22 15:39:10.91321002 +0000 UTC m=+0.943888589 container cleanup d9502dded8f8e7e5ac341e77b97e9582e0d9e4dc0528649f3adc26c17253d3e4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, version=8, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, name=centos-stream, io.openshift.expose-services=) 2024-04-22T15:39:11.110 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:10 smithi145 bash[107828]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2 2024-04-22T15:39:11.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:11 smithi145 ceph-mon[88162]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:39:11.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:11 smithi145 ceph-mon[88162]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:39:11.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:11 smithi145 ceph-mon[88162]: osdmap e119: 8 total, 7 up, 8 in 2024-04-22T15:39:11.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:11 smithi145 ceph-mon[82523]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:39:11.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:11 smithi145 ceph-mon[82523]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:39:11.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:11 smithi145 ceph-mon[82523]: osdmap e119: 8 total, 7 up, 8 in 2024-04-22T15:39:11.373 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:11 smithi145 podman[107845]: 2024-04-22 15:39:11.173151477 +0000 UTC m=+0.647311407 container remove d9502dded8f8e7e5ac341e77b97e9582e0d9e4dc0528649f3adc26c17253d3e4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2, io.buildah.version=1.19.8, architecture=x86_64, ceph=True, GIT_CLEAN=True, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, io.openshift.expose-services=, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, distribution-scope=public, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., name=centos-stream, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-22T15:39:11.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:11 smithi172 ceph-mon[65307]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:39:11.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:11 smithi172 ceph-mon[65307]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:39:11.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:11 smithi172 ceph-mon[65307]: osdmap e119: 8 total, 7 up, 8 in 2024-04-22T15:39:11.848 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:11 smithi145 podman[107941]: 2024-04-22 15:39:11.433850792 +0000 UTC m=+0.018129918 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:39:11.848 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:11 smithi145 podman[107941]: 2024-04-22 15:39:11.532458348 +0000 UTC m=+0.116737506 container create 7179f4f6ddc9caa39ffc248c68475fc1ee5debb17435c3198926ed5972fc3b3e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, ceph=True) 2024-04-22T15:39:11.848 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:11 smithi145 podman[107941]: 2024-04-22 15:39:11.762420195 +0000 UTC m=+0.346699314 container init 7179f4f6ddc9caa39ffc248c68475fc1ee5debb17435c3198926ed5972fc3b3e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2-deactivate, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-04-22T15:39:11.848 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:11 smithi145 podman[107941]: 2024-04-22 15:39:11.765640068 +0000 UTC m=+0.349919184 container start 7179f4f6ddc9caa39ffc248c68475fc1ee5debb17435c3198926ed5972fc3b3e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-22T15:39:11.848 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:11 smithi145 podman[107941]: 2024-04-22 15:39:11.845511561 +0000 UTC m=+0.429790695 container attach 7179f4f6ddc9caa39ffc248c68475fc1ee5debb17435c3198926ed5972fc3b3e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-22T15:39:12.122 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:12 smithi145 podman[107941]: 2024-04-22 15:39:12.062310135 +0000 UTC m=+0.646589255 container died 7179f4f6ddc9caa39ffc248c68475fc1ee5debb17435c3198926ed5972fc3b3e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2-deactivate, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T15:39:12.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:12 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:39:12.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:12 smithi145 ceph-mon[82523]: osdmap e120: 8 total, 7 up, 8 in 2024-04-22T15:39:12.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:12 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:39:12.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:12 smithi145 ceph-mon[88162]: osdmap e120: 8 total, 7 up, 8 in 2024-04-22T15:39:12.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:12 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:39:12.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:12 smithi172 ceph-mon[65307]: osdmap e120: 8 total, 7 up, 8 in 2024-04-22T15:39:13.062 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:12 smithi145 podman[107968]: 2024-04-22 15:39:12.624718523 +0000 UTC m=+0.557416902 container remove 7179f4f6ddc9caa39ffc248c68475fc1ee5debb17435c3198926ed5972fc3b3e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-04-22T15:39:13.062 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:12 smithi145 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.2.service: Deactivated successfully. 2024-04-22T15:39:13.062 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:12 smithi145 systemd[1]: Stopped Ceph osd.2 for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:39:13.062 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:12 smithi145 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.2.service: Consumed 12.232s CPU time. 2024-04-22T15:39:13.062 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:12 smithi145 systemd[1]: Starting Ceph osd.2 for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:39:13.332 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:13 smithi145 ceph-mon[82523]: pgmap v130: 161 pgs: 33 peering, 128 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:39:13.332 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:13 smithi145 ceph-mon[82523]: Health check failed: Reduced data availability: 5 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-04-22T15:39:13.332 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:13 smithi145 ceph-mon[88162]: pgmap v130: 161 pgs: 33 peering, 128 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:39:13.332 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:13 smithi145 ceph-mon[88162]: Health check failed: Reduced data availability: 5 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-04-22T15:39:13.332 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:13 smithi145 podman[108093]: 2024-04-22 15:39:12.960218185 +0000 UTC m=+0.018406025 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:39:13.333 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:13 smithi145 podman[108093]: 2024-04-22 15:39:13.075578883 +0000 UTC m=+0.133766695 container create a497eabfa5caaa1efacf4913002a3a3849cb73a66e96267501797f956def36cf (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2-activate, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, ceph=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.vendor=CentOS) 2024-04-22T15:39:13.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:13 smithi172 ceph-mon[65307]: pgmap v130: 161 pgs: 33 peering, 128 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:39:13.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:13 smithi172 ceph-mon[65307]: Health check failed: Reduced data availability: 5 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-04-22T15:39:13.622 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:13 smithi145 podman[108093]: 2024-04-22 15:39:13.329320067 +0000 UTC m=+0.387507890 container init a497eabfa5caaa1efacf4913002a3a3849cb73a66e96267501797f956def36cf (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2-activate, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T15:39:13.622 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:13 smithi145 podman[108093]: 2024-04-22 15:39:13.332944752 +0000 UTC m=+0.391132575 container start a497eabfa5caaa1efacf4913002a3a3849cb73a66e96267501797f956def36cf (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2-activate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:39:13.623 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:13 smithi145 podman[108093]: 2024-04-22 15:39:13.420493417 +0000 UTC m=+0.478681230 container attach a497eabfa5caaa1efacf4913002a3a3849cb73a66e96267501797f956def36cf (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_CLEAN=True, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-04-22T15:39:14.123 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2-activate[108107]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:39:14.123 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2-activate[108107]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:39:14.123 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:13 smithi145 bash[108093]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:39:14.123 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:13 smithi145 bash[108093]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:39:14.123 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:13 smithi145 bash[108093]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-22T15:39:14.123 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:13 smithi145 bash[108093]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-22T15:39:14.123 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:13 smithi145 bash[108093]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-22T15:39:14.123 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:13 smithi145 bash[108093]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-22T15:39:14.123 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:13 smithi145 bash[108093]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-22T15:39:14.123 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:13 smithi145 bash[108093]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-22T15:39:14.124 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:13 smithi145 bash[108093]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-22T15:39:14.124 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2-activate[108107]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-22T15:39:14.124 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2-activate[108107]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-22T15:39:14.124 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2-activate[108107]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-22T15:39:14.124 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2-activate[108107]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-22T15:39:14.124 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2-activate[108107]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-22T15:39:14.124 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2-activate[108107]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-22T15:39:14.124 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2-activate[108107]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-22T15:39:14.124 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:13 smithi145 podman[108093]: 2024-04-22 15:39:13.832854301 +0000 UTC m=+0.891042122 container died a497eabfa5caaa1efacf4913002a3a3849cb73a66e96267501797f956def36cf (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2-activate, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:39:14.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:14 smithi172 ceph-mon[65307]: Health check failed: Degraded data redundancy: 26/627 objects degraded (4.147%), 4 pgs degraded (PG_DEGRADED) 2024-04-22T15:39:14.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:14 smithi145 ceph-mon[82523]: Health check failed: Degraded data redundancy: 26/627 objects degraded (4.147%), 4 pgs degraded (PG_DEGRADED) 2024-04-22T15:39:14.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:14 smithi145 ceph-mon[88162]: Health check failed: Degraded data redundancy: 26/627 objects degraded (4.147%), 4 pgs degraded (PG_DEGRADED) 2024-04-22T15:39:15.004 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:14 smithi145 podman[108243]: 2024-04-22 15:39:14.658686084 +0000 UTC m=+0.820629039 container remove a497eabfa5caaa1efacf4913002a3a3849cb73a66e96267501797f956def36cf (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2-activate, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416) 2024-04-22T15:39:15.004 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:14 smithi145 podman[108310]: 2024-04-22 15:39:14.776299806 +0000 UTC m=+0.018005281 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:39:15.004 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:14 smithi145 podman[108310]: 2024-04-22 15:39:14.89316426 +0000 UTC m=+0.134869762 container create 6aea4a926fe471881ad9b87309f4bae1ff43c8c78ad22cd1591b8ec46e6b0b1b (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:39:15.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:15 smithi172 ceph-mon[65307]: pgmap v131: 161 pgs: 4 active+undersized+degraded, 33 peering, 10 active+undersized, 114 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 26/627 objects degraded (4.147%) 2024-04-22T15:39:15.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:15 smithi145 ceph-mon[82523]: pgmap v131: 161 pgs: 4 active+undersized+degraded, 33 peering, 10 active+undersized, 114 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 26/627 objects degraded (4.147%) 2024-04-22T15:39:15.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:39:15] "GET /metrics HTTP/1.1" 200 38186 "" "Prometheus/2.43.0" 2024-04-22T15:39:15.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:15 smithi145 ceph-mon[88162]: pgmap v131: 161 pgs: 4 active+undersized+degraded, 33 peering, 10 active+undersized, 114 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 26/627 objects degraded (4.147%) 2024-04-22T15:39:15.623 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:15 smithi145 podman[108310]: 2024-04-22 15:39:15.138021399 +0000 UTC m=+0.379726867 container init 6aea4a926fe471881ad9b87309f4bae1ff43c8c78ad22cd1591b8ec46e6b0b1b (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True) 2024-04-22T15:39:15.623 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:15 smithi145 podman[108310]: 2024-04-22 15:39:15.141199027 +0000 UTC m=+0.382904489 container start 6aea4a926fe471881ad9b87309f4bae1ff43c8c78ad22cd1591b8ec46e6b0b1b (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-04-22T15:39:15.623 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:15 smithi145 ceph-osd[108329]: -- 172.21.15.145:0/391917696 <== mon.1 v2:172.21.15.145:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x559e8fcb5680 con 0x559e8fc7f000 2024-04-22T15:39:15.623 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:15 smithi145 bash[108310]: 6aea4a926fe471881ad9b87309f4bae1ff43c8c78ad22cd1591b8ec46e6b0b1b 2024-04-22T15:39:15.623 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:15 smithi145 systemd[1]: Started Ceph osd.2 for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:39:16.122 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2[108325]: 2024-04-22T15:39:15.689+0000 7f7ef49dd7c0 -1 Falling back to public interface 2024-04-22T15:39:16.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:16 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:16.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:16 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:16.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:16 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:16.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:16 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:39:16.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:16 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:16.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:16 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:16.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:16 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:16.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:16 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:39:16.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:16 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:16.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:16 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:16.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:16 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:16.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:16 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:39:17.389 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:17 smithi145 ceph-mon[88162]: pgmap v132: 161 pgs: 17 active+undersized+degraded, 30 active+undersized, 114 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 60/627 objects degraded (9.569%) 2024-04-22T15:39:17.389 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:17 smithi145 ceph-mon[88162]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 8 pgs peering) 2024-04-22T15:39:17.389 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:17 smithi145 ceph-mon[82523]: pgmap v132: 161 pgs: 17 active+undersized+degraded, 30 active+undersized, 114 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 60/627 objects degraded (9.569%) 2024-04-22T15:39:17.390 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:17 smithi145 ceph-mon[82523]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 8 pgs peering) 2024-04-22T15:39:17.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:17 smithi172 ceph-mon[65307]: pgmap v132: 161 pgs: 17 active+undersized+degraded, 30 active+undersized, 114 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 60/627 objects degraded (9.569%) 2024-04-22T15:39:17.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:17 smithi172 ceph-mon[65307]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 8 pgs peering) 2024-04-22T15:39:18.744 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:18 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2[108325]: 2024-04-22T15:39:18.395+0000 7f7ef49dd7c0 -1 osd.2 0 read_superblock omap replica is missing. 2024-04-22T15:39:18.744 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:18 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2[108325]: 2024-04-22T15:39:18.421+0000 7f7ef49dd7c0 -1 osd.2 118 log_to_monitors true 2024-04-22T15:39:19.584 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:39:19 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2[108325]: 2024-04-22T15:39:19.320+0000 7f7ee5e35700 -1 osd.2 118 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T15:39:19.584 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:19 smithi145 ceph-mon[88162]: pgmap v133: 161 pgs: 17 active+undersized+degraded, 30 active+undersized, 114 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 60/627 objects degraded (9.569%) 2024-04-22T15:39:19.584 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:19 smithi145 ceph-mon[88162]: from='osd.2 [v2:172.21.15.145:6818/269139600,v1:172.21.15.145:6819/269139600]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-22T15:39:19.584 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:19 smithi145 ceph-mon[88162]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-22T15:39:19.585 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:19 smithi145 ceph-mon[88162]: Health check update: Degraded data redundancy: 60/627 objects degraded (9.569%), 17 pgs degraded (PG_DEGRADED) 2024-04-22T15:39:19.585 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:19 smithi145 ceph-mon[82523]: pgmap v133: 161 pgs: 17 active+undersized+degraded, 30 active+undersized, 114 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 60/627 objects degraded (9.569%) 2024-04-22T15:39:19.585 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:19 smithi145 ceph-mon[82523]: from='osd.2 [v2:172.21.15.145:6818/269139600,v1:172.21.15.145:6819/269139600]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-22T15:39:19.585 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:19 smithi145 ceph-mon[82523]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-22T15:39:19.585 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:19 smithi145 ceph-mon[82523]: Health check update: Degraded data redundancy: 60/627 objects degraded (9.569%), 17 pgs degraded (PG_DEGRADED) 2024-04-22T15:39:19.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:19 smithi172 ceph-mon[65307]: pgmap v133: 161 pgs: 17 active+undersized+degraded, 30 active+undersized, 114 active+clean; 585 KiB data, 136 MiB used, 715 GiB / 715 GiB avail; 60/627 objects degraded (9.569%) 2024-04-22T15:39:19.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:19 smithi172 ceph-mon[65307]: from='osd.2 [v2:172.21.15.145:6818/269139600,v1:172.21.15.145:6819/269139600]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-22T15:39:19.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:19 smithi172 ceph-mon[65307]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-22T15:39:19.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:19 smithi172 ceph-mon[65307]: Health check update: Degraded data redundancy: 60/627 objects degraded (9.569%), 17 pgs degraded (PG_DEGRADED) 2024-04-22T15:39:20.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:20 smithi145 ceph-mon[88162]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-22T15:39:20.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:20 smithi145 ceph-mon[88162]: osdmap e121: 8 total, 7 up, 8 in 2024-04-22T15:39:20.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:20 smithi145 ceph-mon[88162]: from='osd.2 [v2:172.21.15.145:6818/269139600,v1:172.21.15.145:6819/269139600]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:39:20.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:20 smithi145 ceph-mon[88162]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:39:20.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:20 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:20.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:20 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:39:20.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:20 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:20.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:20 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:20 smithi145 ceph-mon[82523]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-22T15:39:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:20 smithi145 ceph-mon[82523]: osdmap e121: 8 total, 7 up, 8 in 2024-04-22T15:39:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:20 smithi145 ceph-mon[82523]: from='osd.2 [v2:172.21.15.145:6818/269139600,v1:172.21.15.145:6819/269139600]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:39:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:20 smithi145 ceph-mon[82523]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:39:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:20 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:20 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:39:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:20 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:20 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:20.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:20 smithi172 ceph-mon[65307]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-22T15:39:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:20 smithi172 ceph-mon[65307]: osdmap e121: 8 total, 7 up, 8 in 2024-04-22T15:39:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:20 smithi172 ceph-mon[65307]: from='osd.2 [v2:172.21.15.145:6818/269139600,v1:172.21.15.145:6819/269139600]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:39:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:20 smithi172 ceph-mon[65307]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:39:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:20 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:20 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:39:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:20 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:20 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:21.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:21 smithi145 ceph-mon[88162]: pgmap v135: 161 pgs: 17 active+undersized+degraded, 30 active+undersized, 114 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 573 B/s rd, 0 op/s; 60/627 objects degraded (9.569%) 2024-04-22T15:39:21.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:21 smithi145 ceph-mon[88162]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:39:21.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:21 smithi145 ceph-mon[88162]: osd.2 [v2:172.21.15.145:6818/269139600,v1:172.21.15.145:6819/269139600] boot 2024-04-22T15:39:21.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:21 smithi145 ceph-mon[88162]: osdmap e122: 8 total, 8 up, 8 in 2024-04-22T15:39:21.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:21 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:39:21.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:21 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:21.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:21 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:21.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:21 smithi145 ceph-mon[82523]: pgmap v135: 161 pgs: 17 active+undersized+degraded, 30 active+undersized, 114 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 573 B/s rd, 0 op/s; 60/627 objects degraded (9.569%) 2024-04-22T15:39:21.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:21 smithi145 ceph-mon[82523]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:39:21.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:21 smithi145 ceph-mon[82523]: osd.2 [v2:172.21.15.145:6818/269139600,v1:172.21.15.145:6819/269139600] boot 2024-04-22T15:39:21.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:21 smithi145 ceph-mon[82523]: osdmap e122: 8 total, 8 up, 8 in 2024-04-22T15:39:21.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:21 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:39:21.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:21 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:21.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:21 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:21.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:21 smithi172 ceph-mon[65307]: pgmap v135: 161 pgs: 17 active+undersized+degraded, 30 active+undersized, 114 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 573 B/s rd, 0 op/s; 60/627 objects degraded (9.569%) 2024-04-22T15:39:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:21 smithi172 ceph-mon[65307]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:39:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:21 smithi172 ceph-mon[65307]: osd.2 [v2:172.21.15.145:6818/269139600,v1:172.21.15.145:6819/269139600] boot 2024-04-22T15:39:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:21 smithi172 ceph-mon[65307]: osdmap e122: 8 total, 8 up, 8 in 2024-04-22T15:39:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:21 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T15:39:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:21 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:21 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:22.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:22 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:39:22.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:22 smithi145 ceph-mon[82523]: osdmap e123: 8 total, 8 up, 8 in 2024-04-22T15:39:22.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:22 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:39:22.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:22 smithi145 ceph-mon[88162]: osdmap e123: 8 total, 8 up, 8 in 2024-04-22T15:39:22.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:22 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:39:22.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:22 smithi172 ceph-mon[65307]: osdmap e123: 8 total, 8 up, 8 in 2024-04-22T15:39:23.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:23 smithi145 ceph-mon[88162]: pgmap v138: 161 pgs: 17 active+undersized+degraded, 30 active+undersized, 114 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 60/627 objects degraded (9.569%) 2024-04-22T15:39:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:23 smithi145 ceph-mon[82523]: pgmap v138: 161 pgs: 17 active+undersized+degraded, 30 active+undersized, 114 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 60/627 objects degraded (9.569%) 2024-04-22T15:39:23.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:23 smithi172 ceph-mon[65307]: pgmap v138: 161 pgs: 17 active+undersized+degraded, 30 active+undersized, 114 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 60/627 objects degraded (9.569%) 2024-04-22T15:39:24.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:24 smithi145 ceph-mon[82523]: Health check update: Degraded data redundancy: 34/627 objects degraded (5.423%), 13 pgs degraded (PG_DEGRADED) 2024-04-22T15:39:24.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:24 smithi145 ceph-mon[88162]: Health check update: Degraded data redundancy: 34/627 objects degraded (5.423%), 13 pgs degraded (PG_DEGRADED) 2024-04-22T15:39:24.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:24 smithi172 ceph-mon[65307]: Health check update: Degraded data redundancy: 34/627 objects degraded (5.423%), 13 pgs degraded (PG_DEGRADED) 2024-04-22T15:39:25.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:25 smithi145 ceph-mon[82523]: pgmap v139: 161 pgs: 13 active+undersized+degraded, 24 active+undersized, 124 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 34/627 objects degraded (5.423%) 2024-04-22T15:39:25.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:25.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:25.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:25.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:39:25.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:25.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:39:25.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:25.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:25 smithi145 ceph-mon[88162]: pgmap v139: 161 pgs: 13 active+undersized+degraded, 24 active+undersized, 124 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 34/627 objects degraded (5.423%) 2024-04-22T15:39:25.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:25.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:25.573 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:25.573 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:39:25.573 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:25.573 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:39:25.573 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:25.573 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:39:25] "GET /metrics HTTP/1.1" 200 38035 "" "Prometheus/2.43.0" 2024-04-22T15:39:25.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:25 smithi172 ceph-mon[65307]: pgmap v139: 161 pgs: 13 active+undersized+degraded, 24 active+undersized, 124 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 34/627 objects degraded (5.423%) 2024-04-22T15:39:25.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:25.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:25.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:25.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:39:25.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:25.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:39:25.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:26.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-mon[88162]: pgmap v140: 161 pgs: 10 active+undersized+degraded, 15 active+undersized, 136 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 923 B/s rd, 0 op/s; 31/627 objects degraded (4.944%) 2024-04-22T15:39:26.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-mon[88162]: pgmap v141: 161 pgs: 10 active+undersized+degraded, 15 active+undersized, 136 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 31/627 objects degraded (4.944%) 2024-04-22T15:39:26.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-mon[88162]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:39:26.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-mon[88162]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:39:26.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-mon[82523]: pgmap v140: 161 pgs: 10 active+undersized+degraded, 15 active+undersized, 136 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 923 B/s rd, 0 op/s; 31/627 objects degraded (4.944%) 2024-04-22T15:39:26.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-mon[82523]: pgmap v141: 161 pgs: 10 active+undersized+degraded, 15 active+undersized, 136 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 31/627 objects degraded (4.944%) 2024-04-22T15:39:26.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-mon[82523]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:39:26.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-mon[82523]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:39:26.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:26 smithi172 ceph-mon[65307]: pgmap v140: 161 pgs: 10 active+undersized+degraded, 15 active+undersized, 136 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 923 B/s rd, 0 op/s; 31/627 objects degraded (4.944%) 2024-04-22T15:39:26.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:26 smithi172 ceph-mon[65307]: pgmap v141: 161 pgs: 10 active+undersized+degraded, 15 active+undersized, 136 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 31/627 objects degraded (4.944%) 2024-04-22T15:39:26.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:26 smithi172 ceph-mon[65307]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:39:26.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:26 smithi172 ceph-mon[65307]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:39:27.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:39:26.687+0000 7f17b18ae700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:39:27.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:39:27.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:39:27.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:39:27.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:39:27.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:27.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:27.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:39:27.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:39:27.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:39:27.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:39:27.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:39:27.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:39:27.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:39:27.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:39:27.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:39:27.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:39:27.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:39:27.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:39:27.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:27.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:27.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:39:27.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:39:27.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:39:27.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:39:27.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:39:27.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:39:27.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:39:27.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:39:27.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:39:27.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:39:27.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:39:27.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:39:27.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:39:27.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:39:27.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:39:27.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:39:27.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:39:27.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:39:27.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:39:27.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:39:27.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:39:27.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:39:27.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:27.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:27.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:39:27.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:39:27.128 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:39:27.128 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:39:27.128 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:39:27.128 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:39:27.128 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:39:27.128 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:39:27.128 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:39:27.128 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:39:27.128 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:39:27.128 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:39:27.128 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:27.129 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:26 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:27.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:39:27.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:27.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:27.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:27.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:27.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:27.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:27.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:27.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:27.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:27.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T15:39:27.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T15:39:27.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:39:27.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T15:39:27.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T15:39:27.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T15:39:27.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T15:39:27.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T15:39:27.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T15:39:27.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T15:39:27.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:39:27.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T15:39:27.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T15:39:27.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T15:39:27.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:39:27.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:27.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:27.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:27.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:27.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:27.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:27.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:27.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:27.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:27.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T15:39:27.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T15:39:27.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:39:27.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T15:39:27.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T15:39:27.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T15:39:27.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T15:39:27.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T15:39:27.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T15:39:27.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T15:39:27.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:39:27.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T15:39:27.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T15:39:27.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T15:39:27.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T15:39:27.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T15:39:27.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:39:27.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:39:27.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:39:27.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:39:27.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:39:27.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:39:27.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T15:39:27.628 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T15:39:27.628 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:39:27.628 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:27.628 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:39:27.628 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:27.628 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:39:27.628 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:27.628 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T15:39:27.628 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T15:39:27.628 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:39:27.628 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:39:27.628 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:39:27.628 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:39:27.628 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:39:27.629 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:39:27.629 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T15:39:27.629 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T15:39:27.629 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:39:27.629 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:27.629 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:39:27.629 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:27.629 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:39:27.629 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:27.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:39:27.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:27.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:27.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:27.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:27.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:27.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:27.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:27.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:27.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:27.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T15:39:27.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T15:39:27.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:39:27.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T15:39:27.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T15:39:27.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T15:39:27.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T15:39:27.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T15:39:27.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T15:39:27.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T15:39:27.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:39:27.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T15:39:27.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T15:39:27.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T15:39:27.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T15:39:27.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T15:39:27.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:39:27.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:39:27.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:39:27.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:39:27.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:39:27.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:39:27.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T15:39:27.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T15:39:27.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:39:27.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:27.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:39:27.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:27.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:39:27.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:28.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-mon[82523]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:39:28.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:39:28.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:39:28.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:39:28.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:39:28.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:28.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:28.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:39:28.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:39:28.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:39:28.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:39:28.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:39:28.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:39:28.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:39:28.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:39:28.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:39:28.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:39:28.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:39:28.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:39:28.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:28.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:28.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:39:28.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:39:28.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:39:28.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:39:28.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:39:28.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:39:28.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:39:28.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:39:28.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:39:28.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:39:28.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:39:28.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:39:28.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:39:28.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:39:28.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:39:28.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:39:28.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:39:28.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:39:28.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:39:28.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:39:28.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:39:28.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:39:28.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:28.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:28.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:39:28.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:39:28.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:39:28.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:39:28.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:39:28.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:39:28.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:39:28.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:39:28.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:39:28.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-mon[88162]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:39:28.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:39:28.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:39:28.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:39:28.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:39:28.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:28.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:28.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:39:28.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:39:28.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:39:28.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:39:28.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:39:28.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:39:28.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:39:28.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:39:28.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:39:28.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:39:28.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:39:28.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:39:28.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:28.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:28.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:39:28.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:39:28.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:39:28.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:39:28.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:39:28.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:39:28.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:39:28.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:39:28.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:39:28.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:39:28.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:39:28.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:39:28.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:39:28.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:39:28.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:39:28.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:39:28.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:39:28.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:39:28.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:39:28.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:39:28.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:39:28.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:39:28.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:28.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:28.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:39:28.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:39:28.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:39:28.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:39:28.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:39:28.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:39:28.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:39:28.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:39:28.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:39:28.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:39:28.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:39:28.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:39:28.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:28.632 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:28.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all nvmeof 2024-04-22T15:39:28.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-mon[82523]: Upgrade: Finalizing container_image settings 2024-04-22T15:39:28.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-mon[82523]: Upgrade: Complete! 2024-04-22T15:39:28.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-mon[82523]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 789 B/s rd, 0 op/s 2024-04-22T15:39:28.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-mon[82523]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 934 B/s rd, 0 op/s 2024-04-22T15:39:28.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-mon[82523]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:39:28.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-mon[82523]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:39:28.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-mon[82523]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 31/627 objects degraded (4.944%), 10 pgs degraded) 2024-04-22T15:39:28.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:39:28.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:39:28.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:39:28.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:28.633 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:28.633 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all nvmeof 2024-04-22T15:39:28.633 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-mon[88162]: Upgrade: Finalizing container_image settings 2024-04-22T15:39:28.633 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-mon[88162]: Upgrade: Complete! 2024-04-22T15:39:28.633 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-mon[88162]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 789 B/s rd, 0 op/s 2024-04-22T15:39:28.633 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-mon[88162]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 934 B/s rd, 0 op/s 2024-04-22T15:39:28.633 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-mon[88162]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:39:28.633 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-mon[88162]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:39:28.633 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-mon[88162]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 31/627 objects degraded (4.944%), 10 pgs degraded) 2024-04-22T15:39:28.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:28 smithi172 ceph-mon[65307]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:39:28.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:39:28.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:39:28.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:39:28.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:39:28.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:28.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:28.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:39:28.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:39:28.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:39:28.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:39:28.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:39:28.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:39:28.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:39:28.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:39:28.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:39:28.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:39:28.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:39:28.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:39:28.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:28.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:28.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:39:28.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:39:28.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:39:28.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:39:28.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:39:28.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:39:28.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:39:28.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:39:28.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:39:28.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:39:28.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:39:28.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:39:28.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:39:28.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:39:28.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:39:28.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:39:28.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:39:28.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:39:28.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:39:28.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:39:28.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:39:28.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:39:28.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:28.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:28.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:39:28.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:39:28.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:39:28.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:39:28.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:39:28.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:39:28.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:39:28.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:39:28.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:39:28.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:39:28.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:39:28.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:39:28.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:28.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:28.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:28 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all nvmeof 2024-04-22T15:39:28.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:28 smithi172 ceph-mon[65307]: Upgrade: Finalizing container_image settings 2024-04-22T15:39:28.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:28 smithi172 ceph-mon[65307]: Upgrade: Complete! 2024-04-22T15:39:28.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:28 smithi172 ceph-mon[65307]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 789 B/s rd, 0 op/s 2024-04-22T15:39:28.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:28 smithi172 ceph-mon[65307]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 934 B/s rd, 0 op/s 2024-04-22T15:39:28.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:28 smithi172 ceph-mon[65307]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:39:28.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:28 smithi172 ceph-mon[65307]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:39:28.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:28 smithi172 ceph-mon[65307]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 31/627 objects degraded (4.944%), 10 pgs degraded) 2024-04-22T15:39:29.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:39:28.802+0000 7f17b18ae700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:39:29.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:39:29.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:39:29.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:39:29.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:39:29.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:29.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:29.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:39:29.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:39:29.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:39:29.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:39:29.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:39:29.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:39:29.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:39:29.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:39:29.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:39:29.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:39:29.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:39:29.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:39:29.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:29.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:29.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:39:29.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:39:29.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:39:29.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:39:29.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:39:29.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:39:29.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:39:29.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:39:29.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:39:29.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:39:29.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:39:29.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:39:29.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:39:29.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:39:29.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:39:29.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:39:29.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:39:29.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:39:29.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:39:29.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:39:29.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:39:29.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:39:29.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:29.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:29.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:39:29.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:39:29.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:39:29.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:39:29.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:39:29.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:39:29.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:39:29.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:39:29.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:39:29.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:39:29.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:39:29.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:39:29.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:29.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:28 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:29.704 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:29 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:39:29.705 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:29 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:29.705 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:29 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:39:29.705 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:29 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:29.705 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:29 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:39:29.705 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:29 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:29.705 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:29 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:39:29.705 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:29 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:29.705 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:29 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:39:29.705 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:29 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:29.705 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:29 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:39:29.705 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:29 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:29.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:29 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:39:29.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:29 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:29.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:29 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:39:29.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:29 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:29.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:29 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:39:29.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:29 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:30.433 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-mon[88162]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 759 B/s rd, 0 op/s 2024-04-22T15:39:30.433 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-mon[88162]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:39:30.433 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:39:30.433 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:39:30.433 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:39:30.433 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:39:30.433 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:30.433 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:30.433 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:39:30.433 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:39:30.434 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:39:30.434 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:39:30.434 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:39:30.434 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:39:30.434 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:39:30.434 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:39:30.434 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:39:30.434 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:39:30.434 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:39:30.434 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:39:30.434 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:30.434 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:30.435 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:39:30.435 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:39:30.435 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:39:30.435 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:39:30.435 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:39:30.435 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:39:30.435 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:39:30.435 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:39:30.435 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:39:30.435 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:39:30.435 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:39:30.435 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:39:30.436 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:39:30.436 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:39:30.436 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:39:30.436 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:39:30.436 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:39:30.436 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:39:30.436 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:39:30.436 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:39:30.436 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:39:30.436 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:39:30.437 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:30.437 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:30.437 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:39:30.437 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:39:30.437 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:39:30.437 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:39:30.437 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:39:30.437 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:39:30.437 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:39:30.437 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:39:30.437 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:39:30.437 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:39:30.438 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:39:30.438 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:39:30.438 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:30.438 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:30.438 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-mon[88162]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:39:30.438 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-mon[88162]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:39:30.438 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-mon[88162]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:39:30.438 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-mon[88162]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:39:30.438 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:30.439 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-mon[82523]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 759 B/s rd, 0 op/s 2024-04-22T15:39:30.439 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-mon[82523]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:39:30.439 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:39:30.439 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:39:30.439 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:39:30.439 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:39:30.439 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:30.439 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:30.439 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:39:30.440 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:39:30.440 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:39:30.440 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:39:30.440 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:39:30.440 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:39:30.440 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:39:30.440 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:39:30.440 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:39:30.440 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:39:30.440 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:39:30.440 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:39:30.440 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:30.440 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:30.441 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:39:30.441 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:39:30.441 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:39:30.441 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:39:30.441 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:39:30.441 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:39:30.441 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:39:30.441 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:39:30.441 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:39:30.441 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:39:30.441 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:39:30.441 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:39:30.441 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:39:30.442 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:39:30.442 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:39:30.442 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:39:30.442 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:39:30.442 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:39:30.442 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:39:30.442 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:39:30.442 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:39:30.442 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:39:30.442 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:30.442 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:30.442 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:39:30.443 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:39:30.443 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:39:30.443 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:39:30.443 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:39:30.443 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:39:30.443 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:39:30.443 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:39:30.443 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:39:30.443 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:39:30.443 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:39:30.443 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:39:30.443 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:30.444 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:30.444 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-mon[82523]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:39:30.444 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-mon[82523]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:39:30.444 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-mon[82523]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:39:30.444 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-mon[82523]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:39:30.444 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:30.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:30 smithi172 ceph-mon[65307]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 759 B/s rd, 0 op/s 2024-04-22T15:39:30.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:30 smithi172 ceph-mon[65307]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:39:30.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:39:30.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:39:30.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:39:30.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:39:30.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:30.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:30.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:39:30.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:39:30.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:39:30.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:39:30.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:39:30.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:39:30.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:39:30.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:39:30.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:39:30.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:39:30.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:39:30.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:39:30.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:30.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:30.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:39:30.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:39:30.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:39:30.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:39:30.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:39:30.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:39:30.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:39:30.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:39:30.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:39:30.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:39:30.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:39:30.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:39:30.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:39:30.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:39:30.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:39:30.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:39:30.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:39:30.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:39:30.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:39:30.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:39:30.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:39:30.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:39:30.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:30.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:30.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:39:30.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:39:30.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:39:30.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:39:30.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:39:30.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:39:30.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:39:30.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:39:30.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:39:30.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:39:30.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:39:30.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:39:30.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:30.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:30.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:30 smithi172 ceph-mon[65307]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:39:30.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:30 smithi172 ceph-mon[65307]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:39:30.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:30 smithi172 ceph-mon[65307]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:39:30.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:30 smithi172 ceph-mon[65307]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:39:30.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:30 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:30.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:39:30.665+0000 7f17b18ae700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:39:30.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:39:30.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:39:30.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:39:30.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:39:30.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:30.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:30.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:39:30.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:39:30.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:39:30.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:39:30.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:39:30.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:39:30.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:39:30.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:39:30.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:39:30.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:39:30.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:39:30.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:39:30.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:30.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:30.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:39:30.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:39:30.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:39:30.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:39:30.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:39:30.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:39:30.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:39:30.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:39:30.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:39:30.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:39:30.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:39:30.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:39:30.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:39:30.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:39:30.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:39:30.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:39:30.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:39:30.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:39:30.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:39:30.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:39:30.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:39:30.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:39:30.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:30.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:30.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:39:30.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:39:30.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:39:30.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:39:30.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:39:30.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:39:30.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:39:30.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:39:30.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:39:30.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:39:30.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:39:30.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:39:30.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:30.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:30 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:31 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:39:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:31 smithi172 ceph-mon[65307]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:39:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:39:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:39:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:39:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:39:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:39:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:39:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:39:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:39:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:39:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:39:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:39:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:39:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:39:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:39:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:39:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:39:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:39:31.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:39:31.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:39:31.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:39:31.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:39:31.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:39:31.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:39:31.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:39:31.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:39:31.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:39:31.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:39:31.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:39:31.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:39:31.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:39:31.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:39:31.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:39:31.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:39:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:39:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:39:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:39:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:39:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:39:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:39:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:39:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:39:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:39:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:39:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:39:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:39:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:39:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:39:31.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:39:31.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:39:31.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:39:31.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:31.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:31.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:31 smithi172 ceph-mon[65307]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-04-22T15:39:31.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:31 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:39:31.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:31 smithi145 ceph-mon[82523]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:39:31.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:39:31.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:39:31.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:39:31.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:39:31.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:31.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:31.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:39:31.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:39:31.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:39:31.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:39:31.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:39:31.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:39:31.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:39:31.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:39:31.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:39:31.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:39:31.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:39:31.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:39:31.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:31.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:31.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:39:31.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:39:31.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:39:31.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:39:31.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:39:31.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:39:31.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:39:31.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:39:31.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:39:31.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:39:31.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:39:31.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:39:31.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:39:31.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:39:31.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:39:31.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:39:31.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:39:31.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:39:31.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:39:31.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:39:31.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:39:31.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:39:31.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:31.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:31.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:39:31.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:39:31.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:39:31.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:39:31.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:39:31.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:39:31.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:39:31.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:39:31.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:39:31.876 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:31 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:39:31.876 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:31 smithi145 ceph-mon[88162]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:39:31.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:39:31.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:39:31.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:39:31.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:39:31.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:31.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:31.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:39:31.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:39:31.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:39:31.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:39:31.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:39:31.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:39:31.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:39:31.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:39:31.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:39:31.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:39:31.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:39:31.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:39:31.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:31.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:31.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:39:31.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:39:31.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:39:31.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:39:31.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:39:31.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:39:31.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:39:31.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:39:31.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:39:31.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:39:31.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:39:31.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:39:31.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:39:31.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:39:31.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:39:31.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:39:31.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:39:31.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:39:31.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:39:31.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:39:31.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:39:31.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:39:31.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:31.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:31.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:39:31.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:39:31.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:39:31.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:39:31.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:39:31.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:39:31.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:39:31.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:39:31.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:39:31.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:39:31.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:39:31.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:39:31.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:31.881 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:31.881 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:31 smithi145 ceph-mon[88162]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-04-22T15:39:31.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:39:31.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:39:31.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:39:31.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:39:31.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:39:31.881 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:31 smithi145 ceph-mon[82523]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-04-22T15:39:34.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:33 smithi145 ceph-mon[82523]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 848 B/s rd, 0 op/s 2024-04-22T15:39:34.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:33 smithi145 ceph-mon[88162]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 848 B/s rd, 0 op/s 2024-04-22T15:39:34.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:33 smithi172 ceph-mon[65307]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 848 B/s rd, 0 op/s 2024-04-22T15:39:35.541 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:35.541 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:39:35.541 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:35.541 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:35.542 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:39:35.542 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:35.542 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:39:35] "GET /metrics HTTP/1.1" 200 38195 "" "Prometheus/2.43.0" 2024-04-22T15:39:35.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:35.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:39:35.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:36.247 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch ps' 2024-04-22T15:39:36.555 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:36 smithi145 ceph-mon[82523]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 637 B/s rd, 0 op/s 2024-04-22T15:39:36.555 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:36 smithi145 ceph-mon[88162]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 637 B/s rd, 0 op/s 2024-04-22T15:39:36.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:36 smithi172 ceph-mon[65307]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 637 B/s rd, 0 op/s 2024-04-22T15:39:37.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:37 smithi145 ceph-mon[82523]: from='client.54331 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:39:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:37 smithi145 ceph-mon[88162]: from='client.54331 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:39:37.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:37 smithi172 ceph-mon[65307]: from='client.54331 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:39:38.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:38 smithi145 ceph-mon[88162]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:39:38.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:38 smithi145 ceph-mon[82523]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:39:38.526 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:39:38.526 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (6m) 18s ago 28m 27.0M - 0.25.0 c8568f914cd2 1c58ccf37ffd 2024-04-22T15:39:38.526 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (6m) 3m ago 28m 79.5M - 9.4.12 2bacad6d85d8 2f825d15ab10 2024-04-22T15:39:38.526 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 unknown 18s ago 27m 60.7M - 3.5 e1d6a67b021e 9704631ffa0e 2024-04-22T15:39:38.526 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443,9283,8765 running (7m) 3m ago 33m 448M - 19.0.0-3223-g5883d494 3d983c26e278 e7500ef71604 2024-04-22T15:39:38.526 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (17m) 18s ago 34m 509M - 19.0.0-3223-g5883d494 3d983c26e278 b2de97566cf4 2024-04-22T15:39:38.527 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (3m) 18s ago 35m 51.7M 2048M 19.0.0-3223-g5883d494 3d983c26e278 7bfdc11120f4 2024-04-22T15:39:38.527 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (4m) 3m ago 33m 39.4M 2048M 19.0.0-3223-g5883d494 3d983c26e278 0a360b08a6d6 2024-04-22T15:39:38.527 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (3m) 18s ago 33m 41.1M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38dcfa677c18 2024-04-22T15:39:38.527 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (7m) 18s ago 29m 15.3M - 1.5.0 0da6a335fe13 8c75c26fec05 2024-04-22T15:39:38.527 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (7m) 3m ago 29m 15.9M - 1.5.0 0da6a335fe13 8fd2949e2eaf 2024-04-22T15:39:38.527 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (110s) 18s ago 33m 51.2M 2992M 19.0.0-3223-g5883d494 3d983c26e278 e4bc06317af0 2024-04-22T15:39:38.527 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (74s) 18s ago 32m 43.5M 2992M 19.0.0-3223-g5883d494 3d983c26e278 d299f622f08d 2024-04-22T15:39:38.527 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (23s) 18s ago 32m 12.1M 2992M 19.0.0-3223-g5883d494 3d983c26e278 6aea4a926fe4 2024-04-22T15:39:38.527 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (31m) 18s ago 31m 55.9M 2992M 17.2.0 e1d6a67b021e 8c72f30e54fc 2024-04-22T15:39:38.527 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (31m) 3m ago 31m 57.8M 3248M 17.2.0 e1d6a67b021e 80001f97ed35 2024-04-22T15:39:38.527 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (31m) 3m ago 31m 54.2M 3248M 17.2.0 e1d6a67b021e ec3ab1335b5c 2024-04-22T15:39:38.528 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (30m) 3m ago 30m 54.9M 3248M 17.2.0 e1d6a67b021e 45d475edf1ca 2024-04-22T15:39:38.528 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (30m) 3m ago 30m 53.0M 3248M 17.2.0 e1d6a67b021e cc9e1d9e7916 2024-04-22T15:39:38.528 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (6m) 3m ago 29m 63.9M - 2.43.0 a07b618ecd1d 51cf4df6e8d4 2024-04-22T15:39:38.528 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (28m) 18s ago 28m 79.9M - 17.2.0 e1d6a67b021e c90a636c947e 2024-04-22T15:39:38.528 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (27m) 3m ago 27m 76.5M - 17.2.0 e1d6a67b021e 20ac46e9bfda 2024-04-22T15:39:38.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:38 smithi172 ceph-mon[65307]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:39:39.289 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-04-22T15:39:40.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:40 smithi145 ceph-mon[82523]: from='client.54337 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:39:40.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:40 smithi145 ceph-mon[82523]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:39:40.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:40 smithi145 ceph-mon[88162]: from='client.54337 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:39:40.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:40 smithi145 ceph-mon[88162]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:39:40.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:40 smithi172 ceph-mon[65307]: from='client.54337 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:39:40.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:40 smithi172 ceph-mon[65307]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:39:41.441 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:39:42.163 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 8'"'"'' 2024-04-22T15:39:42.468 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:42 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:39:42.468 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:42 smithi145 ceph-mon[82523]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:39:42.468 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:42 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/4015363585' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:42.468 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:42 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:39:42.468 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:42 smithi145 ceph-mon[88162]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:39:42.468 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:42 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/4015363585' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:42.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:42 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:39:42.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:42 smithi172 ceph-mon[65307]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:39:42.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:42 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/4015363585' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:39:44.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:44 smithi145 ceph-mon[82523]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:39:44.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:44 smithi145 ceph-mon[88162]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:39:44.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:44 smithi172 ceph-mon[65307]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:39:45.490 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:45 smithi145 ceph-mon[82523]: from='client.44412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:39:45.491 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:45 smithi145 ceph-mon[88162]: from='client.44412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:39:45.491 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:39:45] "GET /metrics HTTP/1.1" 200 38195 "" "Prometheus/2.43.0" 2024-04-22T15:39:45.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:45 smithi172 ceph-mon[65307]: from='client.44412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:39:46.359 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:39:46.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:46 smithi145 ceph-mon[82523]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:39:46.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:46 smithi145 ceph-mon[88162]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:39:46.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:46 smithi172 ceph-mon[65307]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:39:46.968 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch upgrade status' 2024-04-22T15:39:48.537 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:48 smithi145 ceph-mon[88162]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:39:48.538 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:48 smithi145 ceph-mon[82523]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:39:48.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:48 smithi172 ceph-mon[65307]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:39:48.984 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:39:48.984 INFO:teuthology.orchestra.run.smithi145.stdout: "target_image": null, 2024-04-22T15:39:48.984 INFO:teuthology.orchestra.run.smithi145.stdout: "in_progress": false, 2024-04-22T15:39:48.984 INFO:teuthology.orchestra.run.smithi145.stdout: "which": "", 2024-04-22T15:39:48.984 INFO:teuthology.orchestra.run.smithi145.stdout: "services_complete": [], 2024-04-22T15:39:48.984 INFO:teuthology.orchestra.run.smithi145.stdout: "progress": null, 2024-04-22T15:39:48.984 INFO:teuthology.orchestra.run.smithi145.stdout: "message": "", 2024-04-22T15:39:48.984 INFO:teuthology.orchestra.run.smithi145.stdout: "is_paused": false 2024-04-22T15:39:48.984 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:39:49.605 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:49 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:39:49.606 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:49 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:39:49.628 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph health detail' 2024-04-22T15:39:49.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:49 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:39:50.452 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:50 smithi145 ceph-mon[88162]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:39:50.452 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:50 smithi145 ceph-mon[88162]: from='client.44418 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:39:50.452 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:50 smithi145 ceph-mon[82523]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:39:50.452 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:50 smithi145 ceph-mon[82523]: from='client.44418 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:39:50.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:50 smithi172 ceph-mon[65307]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:39:50.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:50 smithi172 ceph-mon[65307]: from='client.44418 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:39:51.801 INFO:teuthology.orchestra.run.smithi145.stdout:HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:39:51.801 INFO:teuthology.orchestra.run.smithi145.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:39:51.802 INFO:teuthology.orchestra.run.smithi145.stdout: daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state 2024-04-22T15:39:52.402 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd' 2024-04-22T15:39:52.425 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:52 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:39:52.425 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:52 smithi145 ceph-mon[88162]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:39:52.425 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:52 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/1356576140' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:39:52.425 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:52 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:39:52.425 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:52 smithi145 ceph-mon[82523]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:39:52.425 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:52 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/1356576140' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:39:52.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:52 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:39:52.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:52 smithi172 ceph-mon[65307]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:39:52.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:52 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/1356576140' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:39:54.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:54 smithi145 ceph-mon[88162]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:39:54.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:54 smithi145 ceph-mon[82523]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:39:54.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:54 smithi172 ceph-mon[65307]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:39:55.592 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:55 smithi145 ceph-mon[82523]: from='client.44430 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:39:55.593 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:55 smithi145 ceph-mon[88162]: from='client.44430 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:39:55.593 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:39:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:39:55] "GET /metrics HTTP/1.1" 200 38191 "" "Prometheus/2.43.0" 2024-04-22T15:39:55.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:55 smithi172 ceph-mon[65307]: from='client.44430 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:39:56.484 INFO:teuthology.orchestra.run.smithi145.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:39:56.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:56 smithi145 ceph-mon[82523]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:39:56.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:56 smithi145 ceph-mon[88162]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:39:56.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:56 smithi172 ceph-mon[65307]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:39:57.155 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-22T15:39:57.767 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:57 smithi145 ceph-mon[88162]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:39:57.768 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:57 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:57.768 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:57 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:39:57.768 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:57 smithi145 ceph-mon[88162]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:39:57.768 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:57 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:57.768 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:57 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:39:57.768 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:57 smithi145 ceph-mon[88162]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 998 B/s rd, 0 op/s 2024-04-22T15:39:57.768 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:57 smithi145 ceph-mon[88162]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:39:57.768 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:57 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:57.768 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:57 smithi145 ceph-mon[88162]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:39:57.768 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:57 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:39:57.768 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:57 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:57.768 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:39:57 smithi145 ceph-mon[88162]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:39:57.769 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:57 smithi145 ceph-mon[82523]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:39:57.769 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:57 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:57.769 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:57 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:39:57.769 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:57 smithi145 ceph-mon[82523]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:39:57.769 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:57 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:57.769 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:57 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:39:57.769 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:57 smithi145 ceph-mon[82523]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 998 B/s rd, 0 op/s 2024-04-22T15:39:57.769 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:57 smithi145 ceph-mon[82523]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:39:57.769 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:57 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:57.769 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:57 smithi145 ceph-mon[82523]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:39:57.769 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:57 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:39:57.769 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:57 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:57.769 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:39:57 smithi145 ceph-mon[82523]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:39:57.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:57 smithi172 ceph-mon[65307]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:39:57.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:57 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:57.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:57 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:39:57.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:57 smithi172 ceph-mon[65307]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:39:57.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:57 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:57.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:57 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:39:57.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:57 smithi172 ceph-mon[65307]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 998 B/s rd, 0 op/s 2024-04-22T15:39:57.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:57 smithi172 ceph-mon[65307]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:39:57.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:57 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:39:57.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:57 smithi172 ceph-mon[65307]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:39:57.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:57 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:39:57.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:57 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:39:57.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:39:57 smithi172 ceph-mon[65307]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:40:00.067 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:40:00.318 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-mon[88162]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 620 B/s rd, 0 op/s 2024-04-22T15:40:00.318 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-mon[88162]: overall HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:40:00.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-mon[82523]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 620 B/s rd, 0 op/s 2024-04-22T15:40:00.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-mon[82523]: overall HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:40:00.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:00 smithi172 ceph-mon[65307]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 620 B/s rd, 0 op/s 2024-04-22T15:40:00.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:00 smithi172 ceph-mon[65307]: overall HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:40:00.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:40:00.428+0000 7f17b18ae700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:40:00.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:40:00.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:40:00.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:40:00.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:40:00.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:00.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:00.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:40:00.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:40:00.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:40:00.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:40:00.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:40:00.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:40:00.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:40:00.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:40:00.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:40:00.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:40:00.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:40:00.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:40:00.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:00.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:00.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:40:00.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:40:00.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:40:00.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:40:00.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:40:00.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:40:00.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:40:00.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:40:00.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:40:00.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:40:00.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:40:00.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:40:00.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:40:00.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:40:00.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:40:00.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:40:00.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:40:00.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:40:00.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:40:00.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:40:00.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:40:00.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:40:00.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:00.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:00.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:40:00.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:40:00.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:40:00.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:40:00.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:40:00.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:40:00.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:40:00.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:40:00.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:40:00.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:40:00.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:40:00.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:40:00.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:00.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:00 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:00.769 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:40:00.769 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (6m) 40s ago 29m 27.0M - 0.25.0 c8568f914cd2 1c58ccf37ffd 2024-04-22T15:40:00.769 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (6m) 4m ago 28m 79.5M - 9.4.12 2bacad6d85d8 2f825d15ab10 2024-04-22T15:40:00.770 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 unknown 40s ago 28m 60.7M - 3.5 e1d6a67b021e 9704631ffa0e 2024-04-22T15:40:00.770 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443,9283,8765 running (8m) 4m ago 33m 448M - 19.0.0-3223-g5883d494 3d983c26e278 e7500ef71604 2024-04-22T15:40:00.770 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (18m) 40s ago 35m 509M - 19.0.0-3223-g5883d494 3d983c26e278 b2de97566cf4 2024-04-22T15:40:00.770 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (4m) 40s ago 35m 51.7M 2048M 19.0.0-3223-g5883d494 3d983c26e278 7bfdc11120f4 2024-04-22T15:40:00.770 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (5m) 4m ago 33m 39.4M 2048M 19.0.0-3223-g5883d494 3d983c26e278 0a360b08a6d6 2024-04-22T15:40:00.770 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (3m) 40s ago 34m 41.1M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38dcfa677c18 2024-04-22T15:40:00.770 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (7m) 40s ago 29m 15.3M - 1.5.0 0da6a335fe13 8c75c26fec05 2024-04-22T15:40:00.770 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (7m) 4m ago 29m 15.9M - 1.5.0 0da6a335fe13 8fd2949e2eaf 2024-04-22T15:40:00.770 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (2m) 40s ago 33m 51.2M 2992M 19.0.0-3223-g5883d494 3d983c26e278 e4bc06317af0 2024-04-22T15:40:00.770 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (96s) 40s ago 33m 43.5M 2992M 19.0.0-3223-g5883d494 3d983c26e278 d299f622f08d 2024-04-22T15:40:00.770 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (45s) 40s ago 32m 12.1M 2992M 19.0.0-3223-g5883d494 3d983c26e278 6aea4a926fe4 2024-04-22T15:40:00.770 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (32m) 40s ago 32m 55.9M 2992M 17.2.0 e1d6a67b021e 8c72f30e54fc 2024-04-22T15:40:00.770 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (31m) 4m ago 31m 57.8M 3248M 17.2.0 e1d6a67b021e 80001f97ed35 2024-04-22T15:40:00.770 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (31m) 4m ago 31m 54.2M 3248M 17.2.0 e1d6a67b021e ec3ab1335b5c 2024-04-22T15:40:00.771 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (31m) 4m ago 31m 54.9M 3248M 17.2.0 e1d6a67b021e 45d475edf1ca 2024-04-22T15:40:00.771 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (30m) 4m ago 30m 53.0M 3248M 17.2.0 e1d6a67b021e cc9e1d9e7916 2024-04-22T15:40:00.771 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (7m) 4m ago 29m 63.9M - 2.43.0 a07b618ecd1d 51cf4df6e8d4 2024-04-22T15:40:00.771 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (28m) 40s ago 28m 79.9M - 17.2.0 e1d6a67b021e c90a636c947e 2024-04-22T15:40:00.771 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (28m) 4m ago 28m 76.5M - 17.2.0 e1d6a67b021e 20ac46e9bfda 2024-04-22T15:40:01.203 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:40:01.203 INFO:teuthology.orchestra.run.smithi145.stdout: "mon": { 2024-04-22T15:40:01.203 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 3 2024-04-22T15:40:01.203 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:40:01.204 INFO:teuthology.orchestra.run.smithi145.stdout: "mgr": { 2024-04-22T15:40:01.204 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:40:01.204 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:40:01.204 INFO:teuthology.orchestra.run.smithi145.stdout: "osd": { 2024-04-22T15:40:01.204 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-04-22T15:40:01.204 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 3 2024-04-22T15:40:01.204 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:40:01.204 INFO:teuthology.orchestra.run.smithi145.stdout: "rgw": { 2024-04-22T15:40:01.204 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:40:01.204 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:40:01.204 INFO:teuthology.orchestra.run.smithi145.stdout: "overall": { 2024-04-22T15:40:01.204 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-04-22T15:40:01.204 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 8 2024-04-22T15:40:01.204 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:40:01.205 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:40:01.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:01 smithi172 ceph-mon[65307]: from='client.54373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:40:01.540 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:40:01.540 INFO:teuthology.orchestra.run.smithi145.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", 2024-04-22T15:40:01.540 INFO:teuthology.orchestra.run.smithi145.stdout: "in_progress": true, 2024-04-22T15:40:01.540 INFO:teuthology.orchestra.run.smithi145.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-04-22T15:40:01.540 INFO:teuthology.orchestra.run.smithi145.stdout: "services_complete": [], 2024-04-22T15:40:01.540 INFO:teuthology.orchestra.run.smithi145.stdout: "progress": "", 2024-04-22T15:40:01.540 INFO:teuthology.orchestra.run.smithi145.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa image", 2024-04-22T15:40:01.540 INFO:teuthology.orchestra.run.smithi145.stdout: "is_paused": false 2024-04-22T15:40:01.540 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:40:01.541 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:01 smithi145 ceph-mon[82523]: from='client.54373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:40:01.541 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:01 smithi145 ceph-mon[88162]: from='client.54373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:40:02.422 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:02 smithi145 ceph-mon[82523]: from='client.54379 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:40:02.423 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:02 smithi145 ceph-mon[82523]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:40:02.423 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:40:02.423 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:40:02.423 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:40:02.423 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:40:02.423 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:02.423 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:02.423 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:40:02.423 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:40:02.423 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:40:02.423 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:40:02.423 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:40:02.423 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:40:02.423 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:40:02.424 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:40:02.424 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:40:02.424 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:40:02.424 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:40:02.424 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:40:02.424 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:02.424 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:02.424 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:40:02.424 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:40:02.424 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:40:02.424 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:40:02.424 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:40:02.424 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:40:02.424 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:40:02.424 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:40:02.425 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:40:02.425 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:40:02.425 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:40:02.425 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:40:02.425 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:40:02.425 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:40:02.425 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:40:02.425 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:40:02.425 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:40:02.425 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:40:02.425 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:40:02.425 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:40:02.425 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:40:02.425 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:40:02.425 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:02.426 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:02.426 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:40:02.426 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:40:02.426 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:40:02.426 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:40:02.426 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:40:02.426 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:40:02.426 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:40:02.426 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:40:02.426 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:40:02.426 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:02 smithi145 ceph-mon[88162]: from='client.54379 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:40:02.427 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:02 smithi145 ceph-mon[88162]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:40:02.427 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:40:02.427 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:40:02.427 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:40:02.427 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:40:02.427 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:02.427 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:02.427 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:40:02.427 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:40:02.427 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:40:02.427 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:40:02.427 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:40:02.427 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:40:02.427 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:40:02.427 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:40:02.428 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:40:02.428 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:40:02.428 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:40:02.428 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:40:02.428 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:02.428 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:02.428 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:40:02.428 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:40:02.428 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:40:02.428 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:40:02.428 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:40:02.428 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:40:02.428 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:40:02.428 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:40:02.428 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:40:02.429 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:40:02.429 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:40:02.429 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:40:02.429 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:40:02.429 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:40:02.429 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:40:02.429 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:40:02.429 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:40:02.429 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:40:02.429 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:40:02.429 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:40:02.429 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:40:02.429 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:40:02.429 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:02.430 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:02.430 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:40:02.430 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:40:02.430 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:40:02.430 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:40:02.430 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:40:02.430 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:40:02.430 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:40:02.430 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:40:02.430 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:40:02.430 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:40:02.431 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:40:02.431 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:40:02.431 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:02.431 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:02.431 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:02 smithi145 ceph-mon[82523]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:40:02.431 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:02 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:40:02.431 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:02 smithi145 ceph-mon[82523]: from='client.44448 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:40:02.431 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:02 smithi145 ceph-mon[82523]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:40:02.431 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:02 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/454307868' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:02.431 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:40:02.431 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:40:02.431 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:40:02.432 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:02.432 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:02.432 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:02 smithi145 ceph-mon[88162]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:40:02.432 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:02 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:40:02.432 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:02 smithi145 ceph-mon[88162]: from='client.44448 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:40:02.432 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:02 smithi145 ceph-mon[88162]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:40:02.432 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:02 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/454307868' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:02.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:02 smithi172 ceph-mon[65307]: from='client.54379 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:40:02.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:02 smithi172 ceph-mon[65307]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:40:02.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:40:02.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:40:02.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:40:02.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:40:02.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:02.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:02.468 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:40:02.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:40:02.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:40:02.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:40:02.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:40:02.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:40:02.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:40:02.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:40:02.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:40:02.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:40:02.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:40:02.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:40:02.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:02.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:02.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:40:02.469 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:40:02.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:40:02.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:40:02.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:40:02.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:40:02.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:40:02.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:40:02.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:40:02.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:40:02.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:40:02.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:40:02.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:40:02.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:40:02.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:40:02.470 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:40:02.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:40:02.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:40:02.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:40:02.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:40:02.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:40:02.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:40:02.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:02.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:02.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:40:02.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:40:02.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:40:02.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:40:02.471 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:40:02.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:40:02.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:40:02.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:40:02.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:40:02.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:40:02.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:40:02.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:40:02.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:02.472 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:02.472 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:02 smithi172 ceph-mon[65307]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:40:02.472 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:02 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:40:02.472 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:02 smithi172 ceph-mon[65307]: from='client.44448 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:40:02.472 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:02 smithi172 ceph-mon[65307]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:40:02.472 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:02 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/454307868' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:03.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:03 smithi172 ceph-mon[65307]: from='client.44460 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:40:03.593 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:03 smithi145 ceph-mon[88162]: from='client.44460 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:40:03.594 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:03 smithi145 ceph-mon[82523]: from='client.44460 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:40:04.330 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:04 smithi145 ceph-mon[88162]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:40:04.330 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:04 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:04.331 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:04 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:40:04.331 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:04 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:04.331 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:04 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:04.331 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:04 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:04.331 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:04 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:04.331 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:04 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:04.331 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:04 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:04.331 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:04 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-22T15:40:04.331 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:04 smithi145 ceph-mon[82523]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:40:04.331 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:04 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:04.331 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:04 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:40:04.332 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:04 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:04.332 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:04 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:04.332 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:04 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:04.332 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:04 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:04.332 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:04 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:04.332 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:04 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:04.332 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:04 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-22T15:40:04.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:04 smithi172 ceph-mon[65307]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:40:04.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:04 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:04.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:04 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:40:04.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:04 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:04.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:04 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:04.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:04 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:04.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:04 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:04.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:04 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:04.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:04 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:04.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:04 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-22T15:40:05.270 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:05 smithi145 ceph-mon[88162]: Upgrade: Target is version 19.0.0-3223-g5883d494 (squid) 2024-04-22T15:40:05.270 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:05 smithi145 ceph-mon[88162]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, digests ['quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb'] 2024-04-22T15:40:05.270 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:05 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all mgr 2024-04-22T15:40:05.270 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:05 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all mon 2024-04-22T15:40:05.270 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:05 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all crash 2024-04-22T15:40:05.271 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:05 smithi145 ceph-mon[88162]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-22T15:40:05.271 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:05 smithi145 ceph-mon[88162]: Upgrade: osd.3 is safe to restart 2024-04-22T15:40:05.271 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:05 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:40:05.271 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:05 smithi145 ceph-mon[82523]: Upgrade: Target is version 19.0.0-3223-g5883d494 (squid) 2024-04-22T15:40:05.271 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:05 smithi145 ceph-mon[82523]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, digests ['quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb'] 2024-04-22T15:40:05.271 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:05 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all mgr 2024-04-22T15:40:05.271 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:05 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all mon 2024-04-22T15:40:05.271 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:05 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all crash 2024-04-22T15:40:05.271 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:05 smithi145 ceph-mon[82523]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-22T15:40:05.271 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:05 smithi145 ceph-mon[82523]: Upgrade: osd.3 is safe to restart 2024-04-22T15:40:05.271 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:05 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:40:05.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:05 smithi172 ceph-mon[65307]: Upgrade: Target is version 19.0.0-3223-g5883d494 (squid) 2024-04-22T15:40:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:05 smithi172 ceph-mon[65307]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, digests ['quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb'] 2024-04-22T15:40:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:05 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all mgr 2024-04-22T15:40:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:05 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all mon 2024-04-22T15:40:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:05 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all crash 2024-04-22T15:40:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:05 smithi172 ceph-mon[65307]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-22T15:40:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:05 smithi172 ceph-mon[65307]: Upgrade: osd.3 is safe to restart 2024-04-22T15:40:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:05 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:40:05.622 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:05 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:40:05] "GET /metrics HTTP/1.1" 200 38195 "" "Prometheus/2.43.0" 2024-04-22T15:40:06.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:06 smithi145 ceph-mon[88162]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 620 B/s rd, 0 op/s 2024-04-22T15:40:06.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:06 smithi145 ceph-mon[88162]: Upgrade: Updating osd.3 2024-04-22T15:40:06.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:06 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:06.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:06 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T15:40:06.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:06 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:40:06.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:06 smithi145 ceph-mon[88162]: Deploying daemon osd.3 on smithi145 2024-04-22T15:40:06.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:06 smithi145 ceph-mon[82523]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 620 B/s rd, 0 op/s 2024-04-22T15:40:06.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:06 smithi145 ceph-mon[82523]: Upgrade: Updating osd.3 2024-04-22T15:40:06.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:06 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:06.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:06 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T15:40:06.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:06 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:40:06.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:06 smithi145 ceph-mon[82523]: Deploying daemon osd.3 on smithi145 2024-04-22T15:40:06.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:06 smithi172 ceph-mon[65307]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 620 B/s rd, 0 op/s 2024-04-22T15:40:06.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:06 smithi172 ceph-mon[65307]: Upgrade: Updating osd.3 2024-04-22T15:40:06.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:06 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:06.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:06 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T15:40:06.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:06 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:40:06.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:06 smithi172 ceph-mon[65307]: Deploying daemon osd.3 on smithi145 2024-04-22T15:40:08.513 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:08 smithi145 ceph-mon[88162]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:40:08.514 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:08 smithi145 ceph-mon[82523]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:40:08.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:08 smithi172 ceph-mon[65307]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:40:08.872 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:08 smithi145 systemd[1]: Stopping Ceph osd.3 for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:40:09.319 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3[38329]: 2024-04-22T15:40:08.898+0000 7f6585411700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:40:09.319 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3[38329]: 2024-04-22T15:40:08.898+0000 7f6585411700 -1 osd.3 123 *** Got signal Terminated *** 2024-04-22T15:40:09.319 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:08 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3[38329]: 2024-04-22T15:40:08.898+0000 7f6585411700 -1 osd.3 123 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T15:40:09.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:09 smithi145 ceph-mon[88162]: osd.3 marked itself down and dead 2024-04-22T15:40:09.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:09 smithi145 ceph-mon[82523]: osd.3 marked itself down and dead 2024-04-22T15:40:09.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:09 smithi172 ceph-mon[65307]: osd.3 marked itself down and dead 2024-04-22T15:40:10.122 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:09 smithi145 podman[114967]: 2024-04-22 15:40:09.852064769 +0000 UTC m=+1.200592804 container died 8c72f30e54fc77184feae0806b6384f77dbd57db8462d79732d6fa82060682a3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, ceph=True, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, distribution-scope=public, architecture=x86_64, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-22T15:40:10.473 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:10 smithi145 ceph-mon[82523]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:40:10.473 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:10 smithi145 ceph-mon[82523]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:40:10.473 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:10 smithi145 ceph-mon[82523]: osdmap e124: 8 total, 7 up, 8 in 2024-04-22T15:40:10.474 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:10 smithi145 ceph-mon[88162]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:40:10.474 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:10 smithi145 ceph-mon[88162]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:40:10.474 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:10 smithi145 ceph-mon[88162]: osdmap e124: 8 total, 7 up, 8 in 2024-04-22T15:40:10.474 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:10 smithi145 podman[114967]: 2024-04-22 15:40:10.257386592 +0000 UTC m=+1.605914627 container cleanup 8c72f30e54fc77184feae0806b6384f77dbd57db8462d79732d6fa82060682a3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3, vendor=Red Hat, Inc., io.buildah.version=1.19.8, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, architecture=x86_64, io.openshift.expose-services=, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, GIT_CLEAN=True, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-22T15:40:10.474 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:10 smithi145 bash[114967]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3 2024-04-22T15:40:10.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:10 smithi172 ceph-mon[65307]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 152 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:40:10.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:10 smithi172 ceph-mon[65307]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:40:10.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:10 smithi172 ceph-mon[65307]: osdmap e124: 8 total, 7 up, 8 in 2024-04-22T15:40:10.847 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:10 smithi145 podman[114984]: 2024-04-22 15:40:10.509370089 +0000 UTC m=+0.658656183 container remove 8c72f30e54fc77184feae0806b6384f77dbd57db8462d79732d6fa82060682a3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3, GIT_BRANCH=HEAD, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, io.openshift.expose-services=, version=8, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., RELEASE=HEAD, name=centos-stream, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, release=754, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux ) 2024-04-22T15:40:11.099 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:10 smithi145 podman[115096]: 2024-04-22 15:40:10.748010959 +0000 UTC m=+0.018185748 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:40:11.099 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:10 smithi145 podman[115096]: 2024-04-22 15:40:10.857873968 +0000 UTC m=+0.128048792 container create 97930feb8a16047495d0adbd74daf55a8c54d534448b25397f2fa72f1ab3bc1c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240416) 2024-04-22T15:40:11.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:11 smithi145 ceph-mon[82523]: osdmap e125: 8 total, 7 up, 8 in 2024-04-22T15:40:11.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:11 smithi145 ceph-mon[88162]: osdmap e125: 8 total, 7 up, 8 in 2024-04-22T15:40:11.373 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:11 smithi145 podman[115096]: 2024-04-22 15:40:11.096453404 +0000 UTC m=+0.366628183 container init 97930feb8a16047495d0adbd74daf55a8c54d534448b25397f2fa72f1ab3bc1c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3-deactivate, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T15:40:11.373 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:11 smithi145 podman[115096]: 2024-04-22 15:40:11.099684471 +0000 UTC m=+0.369859256 container start 97930feb8a16047495d0adbd74daf55a8c54d534448b25397f2fa72f1ab3bc1c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_BRANCH=HEAD) 2024-04-22T15:40:11.373 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:11 smithi145 podman[115096]: 2024-04-22 15:40:11.179844364 +0000 UTC m=+0.450019149 container attach 97930feb8a16047495d0adbd74daf55a8c54d534448b25397f2fa72f1ab3bc1c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True) 2024-04-22T15:40:11.677 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:11 smithi145 podman[115096]: 2024-04-22 15:40:11.399881278 +0000 UTC m=+0.670056065 container died 97930feb8a16047495d0adbd74daf55a8c54d534448b25397f2fa72f1ab3bc1c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:40:11.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:11 smithi172 ceph-mon[65307]: osdmap e125: 8 total, 7 up, 8 in 2024-04-22T15:40:12.337 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:11 smithi145 podman[115122]: 2024-04-22 15:40:11.975522384 +0000 UTC m=+0.569994619 container remove 97930feb8a16047495d0adbd74daf55a8c54d534448b25397f2fa72f1ab3bc1c (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3-deactivate, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T15:40:12.338 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:11 smithi145 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.3.service: Deactivated successfully. 2024-04-22T15:40:12.338 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:11 smithi145 systemd[1]: Stopped Ceph osd.3 for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:40:12.338 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:11 smithi145 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.3.service: Consumed 17.302s CPU time. 2024-04-22T15:40:12.338 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:12 smithi145 systemd[1]: Starting Ceph osd.3 for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:40:12.589 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:12 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:40:12.589 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:12 smithi145 ceph-mon[82523]: pgmap v171: 161 pgs: 32 peering, 13 stale+active+clean, 116 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:40:12.589 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:12 smithi145 ceph-mon[82523]: Health check failed: Reduced data availability: 7 pgs inactive, 13 pgs peering (PG_AVAILABILITY) 2024-04-22T15:40:12.589 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:12 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:40:12.589 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:12 smithi145 ceph-mon[88162]: pgmap v171: 161 pgs: 32 peering, 13 stale+active+clean, 116 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:40:12.589 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:12 smithi145 ceph-mon[88162]: Health check failed: Reduced data availability: 7 pgs inactive, 13 pgs peering (PG_AVAILABILITY) 2024-04-22T15:40:12.590 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:12 smithi145 podman[115225]: 2024-04-22 15:40:12.323804437 +0000 UTC m=+0.018193710 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:40:12.590 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:12 smithi145 podman[115225]: 2024-04-22 15:40:12.452152594 +0000 UTC m=+0.146541855 container create 9b2851199382ca9e79cdd09fd314b0f98cb8decdb5af399be72425043178cba6 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3-activate, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-04-22T15:40:12.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:12 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:40:12.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:12 smithi172 ceph-mon[65307]: pgmap v171: 161 pgs: 32 peering, 13 stale+active+clean, 116 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:40:12.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:12 smithi172 ceph-mon[65307]: Health check failed: Reduced data availability: 7 pgs inactive, 13 pgs peering (PG_AVAILABILITY) 2024-04-22T15:40:13.122 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:12 smithi145 podman[115225]: 2024-04-22 15:40:12.696982505 +0000 UTC m=+0.391371775 container init 9b2851199382ca9e79cdd09fd314b0f98cb8decdb5af399be72425043178cba6 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3-activate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T15:40:13.123 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:12 smithi145 podman[115225]: 2024-04-22 15:40:12.700751448 +0000 UTC m=+0.395140709 container start 9b2851199382ca9e79cdd09fd314b0f98cb8decdb5af399be72425043178cba6 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3-activate, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:40:13.123 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:12 smithi145 podman[115225]: 2024-04-22 15:40:12.796611884 +0000 UTC m=+0.491001158 container attach 9b2851199382ca9e79cdd09fd314b0f98cb8decdb5af399be72425043178cba6 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3-activate, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T15:40:13.552 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:13 smithi145 ceph-mon[88162]: pgmap v172: 161 pgs: 5 active+undersized+degraded, 32 peering, 7 stale+active+clean, 9 active+undersized, 108 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 22/627 objects degraded (3.509%) 2024-04-22T15:40:13.552 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:13 smithi145 ceph-mon[88162]: Health check failed: Degraded data redundancy: 22/627 objects degraded (3.509%), 5 pgs degraded (PG_DEGRADED) 2024-04-22T15:40:13.553 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:13 smithi145 ceph-mon[82523]: pgmap v172: 161 pgs: 5 active+undersized+degraded, 32 peering, 7 stale+active+clean, 9 active+undersized, 108 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 22/627 objects degraded (3.509%) 2024-04-22T15:40:13.553 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:13 smithi145 ceph-mon[82523]: Health check failed: Degraded data redundancy: 22/627 objects degraded (3.509%), 5 pgs degraded (PG_DEGRADED) 2024-04-22T15:40:13.553 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3-activate[115239]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:40:13.553 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3-activate[115239]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:40:13.553 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:13 smithi145 bash[115225]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:40:13.553 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:13 smithi145 bash[115225]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:40:13.553 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:13 smithi145 bash[115225]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-22T15:40:13.553 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:13 smithi145 bash[115225]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-22T15:40:13.553 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:13 smithi145 bash[115225]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-22T15:40:13.553 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:13 smithi145 bash[115225]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-22T15:40:13.553 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:13 smithi145 bash[115225]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-22T15:40:13.553 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:13 smithi145 bash[115225]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-22T15:40:13.554 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:13 smithi145 bash[115225]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-22T15:40:13.554 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3-activate[115239]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-22T15:40:13.554 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3-activate[115239]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-22T15:40:13.554 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3-activate[115239]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-22T15:40:13.554 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3-activate[115239]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-22T15:40:13.554 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3-activate[115239]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-22T15:40:13.554 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3-activate[115239]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-22T15:40:13.554 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3-activate[115239]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-22T15:40:13.554 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:13 smithi145 podman[115225]: 2024-04-22 15:40:13.204117698 +0000 UTC m=+0.898506963 container died 9b2851199382ca9e79cdd09fd314b0f98cb8decdb5af399be72425043178cba6 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3-activate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-04-22T15:40:13.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:13 smithi172 ceph-mon[65307]: pgmap v172: 161 pgs: 5 active+undersized+degraded, 32 peering, 7 stale+active+clean, 9 active+undersized, 108 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 22/627 objects degraded (3.509%) 2024-04-22T15:40:13.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:13 smithi172 ceph-mon[65307]: Health check failed: Degraded data redundancy: 22/627 objects degraded (3.509%), 5 pgs degraded (PG_DEGRADED) 2024-04-22T15:40:14.230 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:13 smithi145 podman[115375]: 2024-04-22 15:40:13.976970065 +0000 UTC m=+0.767901032 container remove 9b2851199382ca9e79cdd09fd314b0f98cb8decdb5af399be72425043178cba6 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3-activate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0) 2024-04-22T15:40:14.230 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:14 smithi145 podman[115440]: 2024-04-22 15:40:14.094031949 +0000 UTC m=+0.018643884 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:40:14.491 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:14 smithi145 podman[115440]: 2024-04-22 15:40:14.228025945 +0000 UTC m=+0.152637889 container create c0cd2bdc143986b2cc06e2c160448939217a663c4b9b40e38b7e779f9b379b63 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416) 2024-04-22T15:40:14.872 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:14 smithi145 podman[115440]: 2024-04-22 15:40:14.489343924 +0000 UTC m=+0.413955861 container init c0cd2bdc143986b2cc06e2c160448939217a663c4b9b40e38b7e779f9b379b63 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD) 2024-04-22T15:40:14.872 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:14 smithi145 podman[115440]: 2024-04-22 15:40:14.492379595 +0000 UTC m=+0.416991532 container start c0cd2bdc143986b2cc06e2c160448939217a663c4b9b40e38b7e779f9b379b63 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T15:40:14.873 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:14 smithi145 ceph-osd[115459]: -- 172.21.15.145:0/2584734116 <== mon.1 v2:172.21.15.145:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5609e2a31680 con 0x5609e29fac00 2024-04-22T15:40:14.873 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:14 smithi145 bash[115440]: c0cd2bdc143986b2cc06e2c160448939217a663c4b9b40e38b7e779f9b379b63 2024-04-22T15:40:14.873 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:14 smithi145 systemd[1]: Started Ceph osd.3 for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:40:15.289 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3[115455]: 2024-04-22T15:40:15.036+0000 7fe780b497c0 -1 Falling back to public interface 2024-04-22T15:40:15.571 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:40:15] "GET /metrics HTTP/1.1" 200 38195 "" "Prometheus/2.43.0" 2024-04-22T15:40:15.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:15 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:15.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:15 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:15.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:15 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:40:15.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:15 smithi145 ceph-mon[82523]: pgmap v173: 161 pgs: 14 active+undersized+degraded, 32 peering, 20 active+undersized, 95 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 44/627 objects degraded (7.018%) 2024-04-22T15:40:15.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:15 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:15.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:15 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:15.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:15 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:40:15.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:15 smithi145 ceph-mon[88162]: pgmap v173: 161 pgs: 14 active+undersized+degraded, 32 peering, 20 active+undersized, 95 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 44/627 objects degraded (7.018%) 2024-04-22T15:40:15.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:15 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:15.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:15 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:15.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:15 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:40:15.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:15 smithi172 ceph-mon[65307]: pgmap v173: 161 pgs: 14 active+undersized+degraded, 32 peering, 20 active+undersized, 95 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 44/627 objects degraded (7.018%) 2024-04-22T15:40:17.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:17 smithi145 ceph-mon[82523]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive, 13 pgs peering) 2024-04-22T15:40:17.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:17 smithi145 ceph-mon[88162]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive, 13 pgs peering) 2024-04-22T15:40:17.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:17 smithi172 ceph-mon[65307]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive, 13 pgs peering) 2024-04-22T15:40:18.029 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3[115455]: 2024-04-22T15:40:17.755+0000 7fe780b497c0 -1 osd.3 0 read_superblock omap replica is missing. 2024-04-22T15:40:18.030 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:17 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3[115455]: 2024-04-22T15:40:17.783+0000 7fe780b497c0 -1 osd.3 123 log_to_monitors true 2024-04-22T15:40:18.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:18 smithi145 ceph-mon[88162]: pgmap v174: 161 pgs: 27 active+undersized+degraded, 39 active+undersized, 95 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 82/627 objects degraded (13.078%) 2024-04-22T15:40:18.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:18 smithi145 ceph-mon[88162]: from='osd.3 [v2:172.21.15.145:6826/2093887733,v1:172.21.15.145:6827/2093887733]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-22T15:40:18.373 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:40:18 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3[115455]: 2024-04-22T15:40:18.212+0000 7fe771fa1700 -1 osd.3 123 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T15:40:18.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:18 smithi145 ceph-mon[82523]: pgmap v174: 161 pgs: 27 active+undersized+degraded, 39 active+undersized, 95 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 82/627 objects degraded (13.078%) 2024-04-22T15:40:18.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:18 smithi145 ceph-mon[82523]: from='osd.3 [v2:172.21.15.145:6826/2093887733,v1:172.21.15.145:6827/2093887733]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-22T15:40:18.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:18 smithi172 ceph-mon[65307]: pgmap v174: 161 pgs: 27 active+undersized+degraded, 39 active+undersized, 95 active+clean; 585 KiB data, 153 MiB used, 715 GiB / 715 GiB avail; 82/627 objects degraded (13.078%) 2024-04-22T15:40:18.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:18 smithi172 ceph-mon[65307]: from='osd.3 [v2:172.21.15.145:6826/2093887733,v1:172.21.15.145:6827/2093887733]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-22T15:40:19.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:19 smithi172 ceph-mon[65307]: from='osd.3 [v2:172.21.15.145:6826/2093887733,v1:172.21.15.145:6827/2093887733]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-22T15:40:19.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:19 smithi172 ceph-mon[65307]: osdmap e126: 8 total, 7 up, 8 in 2024-04-22T15:40:19.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:19 smithi172 ceph-mon[65307]: from='osd.3 [v2:172.21.15.145:6826/2093887733,v1:172.21.15.145:6827/2093887733]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:40:19.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:19 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:19.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:19 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:19.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:19 smithi145 ceph-mon[82523]: from='osd.3 [v2:172.21.15.145:6826/2093887733,v1:172.21.15.145:6827/2093887733]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-22T15:40:19.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:19 smithi145 ceph-mon[82523]: osdmap e126: 8 total, 7 up, 8 in 2024-04-22T15:40:19.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:19 smithi145 ceph-mon[82523]: from='osd.3 [v2:172.21.15.145:6826/2093887733,v1:172.21.15.145:6827/2093887733]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:40:19.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:19 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:19.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:19 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:19.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:19 smithi145 ceph-mon[88162]: from='osd.3 [v2:172.21.15.145:6826/2093887733,v1:172.21.15.145:6827/2093887733]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-22T15:40:19.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:19 smithi145 ceph-mon[88162]: osdmap e126: 8 total, 7 up, 8 in 2024-04-22T15:40:19.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:19 smithi145 ceph-mon[88162]: from='osd.3 [v2:172.21.15.145:6826/2093887733,v1:172.21.15.145:6827/2093887733]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-04-22T15:40:19.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:19 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:19.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:19 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:20.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:20 smithi172 ceph-mon[65307]: pgmap v176: 161 pgs: 27 active+undersized+degraded, 39 active+undersized, 95 active+clean; 585 KiB data, 168 MiB used, 715 GiB / 715 GiB avail; 234 B/s rd, 0 op/s; 82/627 objects degraded (13.078%) 2024-04-22T15:40:20.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:20 smithi172 ceph-mon[65307]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:40:20.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:20 smithi172 ceph-mon[65307]: osd.3 [v2:172.21.15.145:6826/2093887733,v1:172.21.15.145:6827/2093887733] boot 2024-04-22T15:40:20.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:20 smithi172 ceph-mon[65307]: osdmap e127: 8 total, 8 up, 8 in 2024-04-22T15:40:20.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:20 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:40:20.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:20 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:20.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:20 smithi172 ceph-mon[65307]: Health check update: Degraded data redundancy: 82/627 objects degraded (13.078%), 27 pgs degraded (PG_DEGRADED) 2024-04-22T15:40:20.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:20 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:40:20.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:20 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:20.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:20 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:20.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:20 smithi172 ceph-mon[65307]: osdmap e128: 8 total, 8 up, 8 in 2024-04-22T15:40:20.482 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:20 smithi145 ceph-mon[82523]: pgmap v176: 161 pgs: 27 active+undersized+degraded, 39 active+undersized, 95 active+clean; 585 KiB data, 168 MiB used, 715 GiB / 715 GiB avail; 234 B/s rd, 0 op/s; 82/627 objects degraded (13.078%) 2024-04-22T15:40:20.483 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:20 smithi145 ceph-mon[82523]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:40:20.483 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:20 smithi145 ceph-mon[82523]: osd.3 [v2:172.21.15.145:6826/2093887733,v1:172.21.15.145:6827/2093887733] boot 2024-04-22T15:40:20.483 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:20 smithi145 ceph-mon[82523]: osdmap e127: 8 total, 8 up, 8 in 2024-04-22T15:40:20.483 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:20 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:40:20.483 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:20 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:20.483 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:20 smithi145 ceph-mon[82523]: Health check update: Degraded data redundancy: 82/627 objects degraded (13.078%), 27 pgs degraded (PG_DEGRADED) 2024-04-22T15:40:20.483 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:20 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:40:20.483 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:20 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:20.483 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:20 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:20.483 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:20 smithi145 ceph-mon[82523]: osdmap e128: 8 total, 8 up, 8 in 2024-04-22T15:40:20.483 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:20 smithi145 ceph-mon[88162]: pgmap v176: 161 pgs: 27 active+undersized+degraded, 39 active+undersized, 95 active+clean; 585 KiB data, 168 MiB used, 715 GiB / 715 GiB avail; 234 B/s rd, 0 op/s; 82/627 objects degraded (13.078%) 2024-04-22T15:40:20.484 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:20 smithi145 ceph-mon[88162]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:40:20.484 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:20 smithi145 ceph-mon[88162]: osd.3 [v2:172.21.15.145:6826/2093887733,v1:172.21.15.145:6827/2093887733] boot 2024-04-22T15:40:20.484 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:20 smithi145 ceph-mon[88162]: osdmap e127: 8 total, 8 up, 8 in 2024-04-22T15:40:20.484 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:20 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T15:40:20.484 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:20 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:20.484 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:20 smithi145 ceph-mon[88162]: Health check update: Degraded data redundancy: 82/627 objects degraded (13.078%), 27 pgs degraded (PG_DEGRADED) 2024-04-22T15:40:20.484 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:20 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:40:20.484 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:20 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:20.484 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:20 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:20.484 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:20 smithi145 ceph-mon[88162]: osdmap e128: 8 total, 8 up, 8 in 2024-04-22T15:40:22.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:22 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:40:22.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:22 smithi145 ceph-mon[88162]: pgmap v179: 161 pgs: 5 peering, 17 active+undersized+degraded, 26 active+undersized, 113 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 59/627 objects degraded (9.410%) 2024-04-22T15:40:22.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:22 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:40:22.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:22 smithi145 ceph-mon[82523]: pgmap v179: 161 pgs: 5 peering, 17 active+undersized+degraded, 26 active+undersized, 113 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 59/627 objects degraded (9.410%) 2024-04-22T15:40:22.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:22 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:40:22.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:22 smithi172 ceph-mon[65307]: pgmap v179: 161 pgs: 5 peering, 17 active+undersized+degraded, 26 active+undersized, 113 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 59/627 objects degraded (9.410%) 2024-04-22T15:40:24.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:24 smithi172 ceph-mon[65307]: pgmap v180: 161 pgs: 5 peering, 14 active+undersized+degraded, 21 active+undersized, 121 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail; 45/627 objects degraded (7.177%) 2024-04-22T15:40:24.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:24 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:24.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:24 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:24.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:24 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:40:24.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:24 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:40:24.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:24 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:24.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:24 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:40:24.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:24 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:40:24.608 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:24 smithi145 ceph-mon[82523]: pgmap v180: 161 pgs: 5 peering, 14 active+undersized+degraded, 21 active+undersized, 121 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail; 45/627 objects degraded (7.177%) 2024-04-22T15:40:24.608 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:24 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:24.608 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:24 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:24.608 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:24 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:40:24.608 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:24 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:40:24.608 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:24 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:24.608 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:24 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:40:24.608 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:24 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:40:24.609 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:24 smithi145 ceph-mon[88162]: pgmap v180: 161 pgs: 5 peering, 14 active+undersized+degraded, 21 active+undersized, 121 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail; 45/627 objects degraded (7.177%) 2024-04-22T15:40:24.609 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:24 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:24.609 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:24 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:24.609 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:24 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:40:24.609 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:24 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:40:24.609 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:24 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:24.609 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:24 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:40:24.609 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:24 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:40:25.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:25 smithi172 ceph-mon[65307]: pgmap v181: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:40:25.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:25 smithi172 ceph-mon[65307]: pgmap v182: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:40:25.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:25 smithi172 ceph-mon[65307]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:40:25.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:25 smithi172 ceph-mon[65307]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:40:25.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:25 smithi172 ceph-mon[65307]: Health check update: Degraded data redundancy: 45/627 objects degraded (7.177%), 14 pgs degraded (PG_DEGRADED) 2024-04-22T15:40:25.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:25 smithi172 ceph-mon[65307]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 45/627 objects degraded (7.177%), 14 pgs degraded) 2024-04-22T15:40:25.622 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:40:25] "GET /metrics HTTP/1.1" 200 38059 "" "Prometheus/2.43.0" 2024-04-22T15:40:25.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-mon[82523]: pgmap v181: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:40:25.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-mon[82523]: pgmap v182: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:40:25.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-mon[82523]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:40:25.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-mon[82523]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:40:25.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-mon[82523]: Health check update: Degraded data redundancy: 45/627 objects degraded (7.177%), 14 pgs degraded (PG_DEGRADED) 2024-04-22T15:40:25.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-mon[82523]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 45/627 objects degraded (7.177%), 14 pgs degraded) 2024-04-22T15:40:25.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-mon[88162]: pgmap v181: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:40:25.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-mon[88162]: pgmap v182: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:40:25.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-mon[88162]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:40:25.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-mon[88162]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:40:25.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-mon[88162]: Health check update: Degraded data redundancy: 45/627 objects degraded (7.177%), 14 pgs degraded (PG_DEGRADED) 2024-04-22T15:40:25.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-mon[88162]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 45/627 objects degraded (7.177%), 14 pgs degraded) 2024-04-22T15:40:25.880 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:40:25.874+0000 7f17b18ae700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:40:25.880 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:40:25.880 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:40:25.880 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:40:25.880 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:40:25.880 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:25.880 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:25.880 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:40:25.880 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:40:25.880 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:40:25.880 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:40:25.880 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:40:25.881 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:40:25.881 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:40:25.881 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:40:25.881 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:40:25.881 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:40:25.881 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:40:25.881 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:40:25.881 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:25.882 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:25.882 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:40:25.882 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:40:25.882 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:40:25.882 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:40:25.882 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:40:25.882 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:40:25.882 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:40:25.882 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:40:25.882 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:40:25.882 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:40:25.882 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:40:25.883 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:40:25.883 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:40:25.883 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:40:25.883 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:40:25.883 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:40:25.883 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:40:25.883 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:40:25.883 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:40:25.883 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:40:25.883 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:40:25.883 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:40:25.883 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:25.883 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:25.883 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:40:25.883 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:40:25.883 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:40:25.884 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:40:25.884 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:40:25.884 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:40:25.884 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:40:25.884 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:40:25.884 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:40:25.884 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:40:25.884 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:40:25.884 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:40:25.884 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:25.884 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:26.474 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:26 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:40:26.474 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:26 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:26.474 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:26 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:26.474 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:26 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:26.474 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:26 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T15:40:26.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:26 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:40:26.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:26 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:26.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:26 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:26.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:26 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:26.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:26 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T15:40:26.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:26 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:40:26.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:26 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:26.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:26 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:26.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:26 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:26.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:26 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T15:40:27.490 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:27 smithi172 ceph-mon[65307]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:40:27.490 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:27 smithi172 ceph-mon[65307]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:40:27.491 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:40:27.491 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:40:27.491 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:40:27.491 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:40:27.491 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:27.491 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:27.491 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:40:27.491 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:40:27.491 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:40:27.491 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:40:27.491 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:40:27.491 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:40:27.491 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:40:27.491 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:40:27.492 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:40:27.492 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:40:27.492 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:40:27.492 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:40:27.492 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:27.492 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:27.492 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:40:27.492 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:40:27.492 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:40:27.492 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:40:27.492 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:40:27.492 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:40:27.492 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:40:27.492 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:40:27.493 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:40:27.493 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:40:27.493 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:40:27.493 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:40:27.493 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:40:27.493 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:40:27.493 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:40:27.493 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:40:27.493 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:40:27.493 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:40:27.493 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:40:27.493 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:40:27.493 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:40:27.493 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:40:27.493 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:27.494 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:27.494 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:40:27.494 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:40:27.494 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:40:27.494 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:40:27.494 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:40:27.494 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:40:27.494 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:40:27.494 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:40:27.494 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:40:27.494 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:40:27.494 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:40:27.494 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:40:27.494 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:27.494 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:27.495 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:27 smithi172 ceph-mon[65307]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T15:40:27.495 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:27 smithi172 ceph-mon[65307]: Upgrade: osd.4 is safe to restart 2024-04-22T15:40:27.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:27 smithi145 ceph-mon[82523]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:40:27.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:27 smithi145 ceph-mon[82523]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:40:27.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:40:27.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:40:27.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:40:27.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:40:27.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:27.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:27.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:40:27.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:40:27.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:40:27.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:40:27.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:40:27.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:40:27.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:40:27.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:40:27.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:40:27.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:40:27.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:40:27.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:40:27.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:27.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:27.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:40:27.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:40:27.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:40:27.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:40:27.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:40:27.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:40:27.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:40:27.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:40:27.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:40:27.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:40:27.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:40:27.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:40:27.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:40:27.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:40:27.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:40:27.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:40:27.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:40:27.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:40:27.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:40:27.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:40:27.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:40:27.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:40:27.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:27.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:27.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:40:27.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:40:27.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:40:27.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:40:27.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:40:27.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:40:27.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:40:27.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:40:27.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:40:27.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:27 smithi145 ceph-mon[88162]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:40:27.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:27 smithi145 ceph-mon[88162]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:40:27.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:40:27.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:40:27.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:40:27.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:40:27.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:27.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:27.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:40:27.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:40:27.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:40:27.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:40:27.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:40:27.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:40:27.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:40:27.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:40:27.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:40:27.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:40:27.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:40:27.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:40:27.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:27.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:27.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:40:27.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:40:27.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:40:27.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:40:27.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:40:27.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:40:27.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:40:27.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:40:27.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:40:27.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:40:27.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:40:27.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:40:27.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:40:27.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:40:27.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:40:27.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:40:27.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:40:27.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:40:27.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:40:27.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:40:27.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:40:27.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:40:27.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:27.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:27.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:40:27.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:40:27.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:40:27.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:40:27.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:40:27.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:40:27.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:40:27.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:40:27.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:40:27.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:40:27.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:40:27.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:40:27.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:27.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:27.631 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:27 smithi145 ceph-mon[88162]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T15:40:27.631 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:27 smithi145 ceph-mon[88162]: Upgrade: osd.4 is safe to restart 2024-04-22T15:40:27.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:40:27.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:40:27.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:40:27.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:27.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:27.631 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:27 smithi145 ceph-mon[82523]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-22T15:40:27.631 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:27 smithi145 ceph-mon[82523]: Upgrade: osd.4 is safe to restart 2024-04-22T15:40:28.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:28 smithi172 ceph-mon[65307]: Upgrade: Updating osd.4 2024-04-22T15:40:28.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:28 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:28.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:28 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T15:40:28.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:28 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:40:28.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:28 smithi172 ceph-mon[65307]: Deploying daemon osd.4 on smithi172 2024-04-22T15:40:28.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:28 smithi172 ceph-mon[65307]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:40:28.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:28 smithi145 ceph-mon[82523]: Upgrade: Updating osd.4 2024-04-22T15:40:28.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:28 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:28.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:28 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T15:40:28.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:28 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:40:28.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:28 smithi145 ceph-mon[82523]: Deploying daemon osd.4 on smithi172 2024-04-22T15:40:28.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:28 smithi145 ceph-mon[82523]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:40:28.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:28 smithi145 ceph-mon[88162]: Upgrade: Updating osd.4 2024-04-22T15:40:28.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:28 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:28.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:28 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T15:40:28.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:28 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:40:28.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:28 smithi145 ceph-mon[88162]: Deploying daemon osd.4 on smithi172 2024-04-22T15:40:28.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:28 smithi145 ceph-mon[88162]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:40:30.718 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:30 smithi172 systemd[1]: Stopping Ceph osd.4 for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:40:31.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:30 smithi172 ceph-mon[65307]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail; 349 B/s rd, 0 op/s 2024-04-22T15:40:31.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:30 smithi172 ceph-mon[65307]: osd.4 marked itself down and dead 2024-04-22T15:40:31.218 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:30 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4[26200]: 2024-04-22T15:40:30.794+0000 7fb559cb8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:40:31.218 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:30 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4[26200]: 2024-04-22T15:40:30.794+0000 7fb559cb8700 -1 osd.4 128 *** Got signal Terminated *** 2024-04-22T15:40:31.218 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:30 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4[26200]: 2024-04-22T15:40:30.794+0000 7fb559cb8700 -1 osd.4 128 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T15:40:31.292 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:30 smithi145 ceph-mon[88162]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail; 349 B/s rd, 0 op/s 2024-04-22T15:40:31.292 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:30 smithi145 ceph-mon[88162]: osd.4 marked itself down and dead 2024-04-22T15:40:31.293 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:30 smithi145 ceph-mon[82523]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail; 349 B/s rd, 0 op/s 2024-04-22T15:40:31.293 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:30 smithi145 ceph-mon[82523]: osd.4 marked itself down and dead 2024-04-22T15:40:31.703 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:31 smithi172 podman[75764]: 2024-04-22 15:40:31.435359668 +0000 UTC m=+0.756890827 container died 80001f97ed351ee30c7c04cbbc4d9f88ca4006d6954f00b92140534173107e41 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, name=centos-stream, vcs-type=git, GIT_BRANCH=HEAD, io.openshift.expose-services=, GIT_CLEAN=True, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, distribution-scope=public) 2024-04-22T15:40:31.946 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:40:31.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:31 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:40:31.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:31 smithi172 ceph-mon[65307]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:40:31.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:31 smithi172 ceph-mon[65307]: osdmap e129: 8 total, 7 up, 8 in 2024-04-22T15:40:31.968 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:31 smithi172 podman[75764]: 2024-04-22 15:40:31.794482024 +0000 UTC m=+1.116013181 container cleanup 80001f97ed351ee30c7c04cbbc4d9f88ca4006d6954f00b92140534173107e41 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4, version=8, name=centos-stream, vendor=Red Hat, Inc., ceph=True, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, io.openshift.expose-services=, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , architecture=x86_64, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-22T15:40:31.968 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:31 smithi172 bash[75764]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4 2024-04-22T15:40:32.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:31 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:40:32.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:31 smithi145 ceph-mon[88162]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:40:32.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:31 smithi145 ceph-mon[88162]: osdmap e129: 8 total, 7 up, 8 in 2024-04-22T15:40:32.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:31 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:40:32.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:31 smithi145 ceph-mon[82523]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:40:32.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:31 smithi145 ceph-mon[82523]: osdmap e129: 8 total, 7 up, 8 in 2024-04-22T15:40:32.457 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:32 smithi172 podman[75791]: 2024-04-22 15:40:32.096098944 +0000 UTC m=+0.655747357 container remove 80001f97ed351ee30c7c04cbbc4d9f88ca4006d6954f00b92140534173107e41 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, architecture=x86_64, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, release=754) 2024-04-22T15:40:32.617 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:40:32.617 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (7m) 13s ago 29m 27.0M - 0.25.0 c8568f914cd2 1c58ccf37ffd 2024-04-22T15:40:32.617 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (7m) 4m ago 29m 79.5M - 9.4.12 2bacad6d85d8 2f825d15ab10 2024-04-22T15:40:32.617 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 unknown 13s ago 28m 60.7M - 3.5 e1d6a67b021e 9704631ffa0e 2024-04-22T15:40:32.617 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443,9283,8765 running (8m) 4m ago 34m 448M - 19.0.0-3223-g5883d494 3d983c26e278 e7500ef71604 2024-04-22T15:40:32.617 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (18m) 13s ago 35m 511M - 19.0.0-3223-g5883d494 3d983c26e278 b2de97566cf4 2024-04-22T15:40:32.617 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (4m) 13s ago 35m 53.7M 2048M 19.0.0-3223-g5883d494 3d983c26e278 7bfdc11120f4 2024-04-22T15:40:32.617 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (5m) 4m ago 34m 39.4M 2048M 19.0.0-3223-g5883d494 3d983c26e278 0a360b08a6d6 2024-04-22T15:40:32.617 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (4m) 13s ago 34m 42.0M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38dcfa677c18 2024-04-22T15:40:32.617 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (8m) 13s ago 30m 15.3M - 1.5.0 0da6a335fe13 8c75c26fec05 2024-04-22T15:40:32.617 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (8m) 4m ago 30m 15.9M - 1.5.0 0da6a335fe13 8fd2949e2eaf 2024-04-22T15:40:32.617 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (2m) 13s ago 33m 51.7M 2992M 19.0.0-3223-g5883d494 3d983c26e278 e4bc06317af0 2024-04-22T15:40:32.618 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (2m) 13s ago 33m 44.0M 2992M 19.0.0-3223-g5883d494 3d983c26e278 d299f622f08d 2024-04-22T15:40:32.618 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (77s) 13s ago 33m 41.7M 2992M 19.0.0-3223-g5883d494 3d983c26e278 6aea4a926fe4 2024-04-22T15:40:32.618 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (18s) 13s ago 32m 15.1M 2992M 19.0.0-3223-g5883d494 3d983c26e278 c0cd2bdc1439 2024-04-22T15:40:32.618 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (32m) 4m ago 32m 57.8M 3248M 17.2.0 e1d6a67b021e 80001f97ed35 2024-04-22T15:40:32.618 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (32m) 4m ago 32m 54.2M 3248M 17.2.0 e1d6a67b021e ec3ab1335b5c 2024-04-22T15:40:32.618 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (31m) 4m ago 31m 54.9M 3248M 17.2.0 e1d6a67b021e 45d475edf1ca 2024-04-22T15:40:32.618 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (31m) 4m ago 31m 53.0M 3248M 17.2.0 e1d6a67b021e cc9e1d9e7916 2024-04-22T15:40:32.618 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (7m) 4m ago 30m 63.9M - 2.43.0 a07b618ecd1d 51cf4df6e8d4 2024-04-22T15:40:32.618 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (28m) 13s ago 28m 80.2M - 17.2.0 e1d6a67b021e c90a636c947e 2024-04-22T15:40:32.618 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (28m) 4m ago 28m 76.5M - 17.2.0 e1d6a67b021e 20ac46e9bfda 2024-04-22T15:40:32.717 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:32 smithi172 podman[75888]: 2024-04-22 15:40:32.357935047 +0000 UTC m=+0.019061132 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:40:32.718 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:32 smithi172 podman[75888]: 2024-04-22 15:40:32.488619131 +0000 UTC m=+0.149745206 container create 44b84ec31542bfef96876b72ae415246967d822e12169a0ed653050a9c74f181 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=) 2024-04-22T15:40:33.014 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:40:33.015 INFO:teuthology.orchestra.run.smithi145.stdout: "mon": { 2024-04-22T15:40:33.015 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 3 2024-04-22T15:40:33.015 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:40:33.015 INFO:teuthology.orchestra.run.smithi145.stdout: "mgr": { 2024-04-22T15:40:33.015 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:40:33.015 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:40:33.015 INFO:teuthology.orchestra.run.smithi145.stdout: "osd": { 2024-04-22T15:40:33.015 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-04-22T15:40:33.015 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 4 2024-04-22T15:40:33.015 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:40:33.015 INFO:teuthology.orchestra.run.smithi145.stdout: "rgw": { 2024-04-22T15:40:33.015 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:40:33.015 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:40:33.015 INFO:teuthology.orchestra.run.smithi145.stdout: "overall": { 2024-04-22T15:40:33.015 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-04-22T15:40:33.016 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 9 2024-04-22T15:40:33.016 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:40:33.016 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:40:33.050 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:32 smithi172 ceph-mon[65307]: pgmap v187: 161 pgs: 23 stale+active+clean, 138 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-22T15:40:33.050 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:32 smithi172 ceph-mon[65307]: from='client.44472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:40:33.050 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:32 smithi172 ceph-mon[65307]: osdmap e130: 8 total, 7 up, 8 in 2024-04-22T15:40:33.050 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:32 smithi172 ceph-mon[65307]: from='client.54415 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:40:33.050 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:32 smithi172 podman[75888]: 2024-04-22 15:40:32.749763937 +0000 UTC m=+0.410890013 container init 44b84ec31542bfef96876b72ae415246967d822e12169a0ed653050a9c74f181 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-04-22T15:40:33.050 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:32 smithi172 podman[75888]: 2024-04-22 15:40:32.753023757 +0000 UTC m=+0.414149838 container start 44b84ec31542bfef96876b72ae415246967d822e12169a0ed653050a9c74f181 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-04-22T15:40:33.050 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:32 smithi172 podman[75888]: 2024-04-22 15:40:32.824872662 +0000 UTC m=+0.485998741 container attach 44b84ec31542bfef96876b72ae415246967d822e12169a0ed653050a9c74f181 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-04-22T15:40:33.339 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:33 smithi172 podman[75888]: 2024-04-22 15:40:33.048316654 +0000 UTC m=+0.709442731 container died 44b84ec31542bfef96876b72ae415246967d822e12169a0ed653050a9c74f181 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:40:33.352 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:40:33.352 INFO:teuthology.orchestra.run.smithi145.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:40:33.352 INFO:teuthology.orchestra.run.smithi145.stdout: "in_progress": true, 2024-04-22T15:40:33.352 INFO:teuthology.orchestra.run.smithi145.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-04-22T15:40:33.352 INFO:teuthology.orchestra.run.smithi145.stdout: "services_complete": [], 2024-04-22T15:40:33.352 INFO:teuthology.orchestra.run.smithi145.stdout: "progress": "4/8 daemons upgraded", 2024-04-22T15:40:33.352 INFO:teuthology.orchestra.run.smithi145.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T15:40:33.352 INFO:teuthology.orchestra.run.smithi145.stdout: "is_paused": false 2024-04-22T15:40:33.352 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:40:33.353 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:32 smithi145 ceph-mon[88162]: pgmap v187: 161 pgs: 23 stale+active+clean, 138 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-22T15:40:33.353 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:32 smithi145 ceph-mon[88162]: from='client.44472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:40:33.353 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:32 smithi145 ceph-mon[88162]: osdmap e130: 8 total, 7 up, 8 in 2024-04-22T15:40:33.353 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:32 smithi145 ceph-mon[88162]: from='client.54415 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:40:33.353 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:32 smithi145 ceph-mon[82523]: pgmap v187: 161 pgs: 23 stale+active+clean, 138 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-22T15:40:33.353 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:32 smithi145 ceph-mon[82523]: from='client.44472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:40:33.353 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:32 smithi145 ceph-mon[82523]: osdmap e130: 8 total, 7 up, 8 in 2024-04-22T15:40:33.353 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:32 smithi145 ceph-mon[82523]: from='client.54415 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:40:33.952 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:33 smithi172 podman[75914]: 2024-04-22 15:40:33.603535505 +0000 UTC m=+0.549808177 container remove 44b84ec31542bfef96876b72ae415246967d822e12169a0ed653050a9c74f181 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-04-22T15:40:33.953 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:33 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.4.service: Deactivated successfully. 2024-04-22T15:40:33.953 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:33 smithi172 systemd[1]: Stopped Ceph osd.4 for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:40:33.953 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:33 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.4.service: Consumed 17.339s CPU time. 2024-04-22T15:40:33.953 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:33 smithi172 systemd[1]: Starting Ceph osd.4 for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:40:34.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:33 smithi172 ceph-mon[65307]: from='client.54421 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:40:34.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:33 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/1405435524' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:34.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:33 smithi172 ceph-mon[65307]: from='client.44496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:40:34.218 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:34 smithi172 podman[76020]: 2024-04-22 15:40:33.948201047 +0000 UTC m=+0.020481100 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:40:34.218 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:34 smithi172 podman[76020]: 2024-04-22 15:40:34.071662443 +0000 UTC m=+0.143942470 container create 75c36ddad68126463988fdb294f2b4b809b6a789565e598d39de33fe4039c6ef (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-04-22T15:40:34.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:33 smithi145 ceph-mon[88162]: from='client.54421 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:40:34.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:33 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/1405435524' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:34.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:33 smithi145 ceph-mon[88162]: from='client.44496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:40:34.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:33 smithi145 ceph-mon[82523]: from='client.54421 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:40:34.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:33 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/1405435524' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:34.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:33 smithi145 ceph-mon[82523]: from='client.44496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:40:34.717 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:34 smithi172 podman[76020]: 2024-04-22 15:40:34.257807641 +0000 UTC m=+0.330087667 container init 75c36ddad68126463988fdb294f2b4b809b6a789565e598d39de33fe4039c6ef (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4-activate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240416) 2024-04-22T15:40:34.718 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:34 smithi172 podman[76020]: 2024-04-22 15:40:34.261066039 +0000 UTC m=+0.333346074 container start 75c36ddad68126463988fdb294f2b4b809b6a789565e598d39de33fe4039c6ef (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4-activate, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-04-22T15:40:34.718 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:34 smithi172 podman[76020]: 2024-04-22 15:40:34.32421256 +0000 UTC m=+0.396492592 container attach 75c36ddad68126463988fdb294f2b4b809b6a789565e598d39de33fe4039c6ef (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4-activate, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416) 2024-04-22T15:40:35.077 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:34 smithi172 ceph-mon[65307]: pgmap v189: 161 pgs: 11 active+undersized+degraded, 12 stale+active+clean, 19 active+undersized, 119 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail; 43/627 objects degraded (6.858%) 2024-04-22T15:40:35.077 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:34 smithi172 ceph-mon[65307]: Health check failed: Degraded data redundancy: 43/627 objects degraded (6.858%), 11 pgs degraded (PG_DEGRADED) 2024-04-22T15:40:35.077 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:35.077 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:40:35.077 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:35.077 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:34 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4-activate[76034]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:40:35.078 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:34 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4-activate[76034]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:40:35.078 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:34 smithi172 bash[76020]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:40:35.078 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:34 smithi172 bash[76020]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:40:35.078 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:34 smithi172 bash[76020]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-22T15:40:35.078 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:34 smithi172 bash[76020]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-22T15:40:35.078 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:34 smithi172 bash[76020]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-22T15:40:35.078 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:34 smithi172 bash[76020]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-22T15:40:35.078 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:34 smithi172 bash[76020]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-04-22T15:40:35.078 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:34 smithi172 bash[76020]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-22T15:40:35.078 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:34 smithi172 bash[76020]: --> ceph-volume raw activate successful for osd ID: 4 2024-04-22T15:40:35.078 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:34 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4-activate[76034]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-22T15:40:35.078 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:34 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4-activate[76034]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-22T15:40:35.078 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:34 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4-activate[76034]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-22T15:40:35.079 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:34 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4-activate[76034]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-22T15:40:35.079 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:34 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4-activate[76034]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-04-22T15:40:35.079 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:34 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4-activate[76034]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-22T15:40:35.079 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:34 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4-activate[76034]: --> ceph-volume raw activate successful for osd ID: 4 2024-04-22T15:40:35.079 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:34 smithi172 podman[76020]: 2024-04-22 15:40:34.757548836 +0000 UTC m=+0.829828867 container died 75c36ddad68126463988fdb294f2b4b809b6a789565e598d39de33fe4039c6ef (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4-activate, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416) 2024-04-22T15:40:35.290 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:34 smithi145 ceph-mon[88162]: pgmap v189: 161 pgs: 11 active+undersized+degraded, 12 stale+active+clean, 19 active+undersized, 119 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail; 43/627 objects degraded (6.858%) 2024-04-22T15:40:35.290 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:34 smithi145 ceph-mon[88162]: Health check failed: Degraded data redundancy: 43/627 objects degraded (6.858%), 11 pgs degraded (PG_DEGRADED) 2024-04-22T15:40:35.290 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:35.290 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:40:35.290 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:35.293 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:34 smithi145 ceph-mon[82523]: pgmap v189: 161 pgs: 11 active+undersized+degraded, 12 stale+active+clean, 19 active+undersized, 119 active+clean; 585 KiB data, 169 MiB used, 715 GiB / 715 GiB avail; 43/627 objects degraded (6.858%) 2024-04-22T15:40:35.293 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:34 smithi145 ceph-mon[82523]: Health check failed: Degraded data redundancy: 43/627 objects degraded (6.858%), 11 pgs degraded (PG_DEGRADED) 2024-04-22T15:40:35.293 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:35.293 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:40:35.293 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:35.622 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:40:35] "GET /metrics HTTP/1.1" 200 38200 "" "Prometheus/2.43.0" 2024-04-22T15:40:35.686 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:35 smithi172 podman[76156]: 2024-04-22 15:40:35.345025602 +0000 UTC m=+0.581819335 container remove 75c36ddad68126463988fdb294f2b4b809b6a789565e598d39de33fe4039c6ef (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4-activate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T15:40:35.687 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:35 smithi172 podman[76225]: 2024-04-22 15:40:35.463368744 +0000 UTC m=+0.018572906 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:40:35.687 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:35 smithi172 podman[76225]: 2024-04-22 15:40:35.579220685 +0000 UTC m=+0.134424847 container create 7ac5776bc07a29bde3f418fa8503a5fbb64dbf4c7c2e9a935e7a1f9889ca5842 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-04-22T15:40:36.217 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:35 smithi172 podman[76225]: 2024-04-22 15:40:35.807374293 +0000 UTC m=+0.362578462 container init 7ac5776bc07a29bde3f418fa8503a5fbb64dbf4c7c2e9a935e7a1f9889ca5842 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T15:40:36.218 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:35 smithi172 podman[76225]: 2024-04-22 15:40:35.81066699 +0000 UTC m=+0.365871161 container start 7ac5776bc07a29bde3f418fa8503a5fbb64dbf4c7c2e9a935e7a1f9889ca5842 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T15:40:36.218 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:35 smithi172 ceph-osd[76243]: -- 172.21.15.172:0/3637779857 <== mon.2 v2:172.21.15.172:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x556c4b245680 con 0x556c4b20ec00 2024-04-22T15:40:36.218 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:35 smithi172 bash[76225]: 7ac5776bc07a29bde3f418fa8503a5fbb64dbf4c7c2e9a935e7a1f9889ca5842 2024-04-22T15:40:36.218 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:35 smithi172 systemd[1]: Started Ceph osd.4 for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:40:36.717 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:36 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4[76239]: 2024-04-22T15:40:36.357+0000 7ff5c3e187c0 -1 Falling back to public interface 2024-04-22T15:40:37.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:36 smithi172 ceph-mon[65307]: pgmap v190: 161 pgs: 27 active+undersized+degraded, 41 active+undersized, 93 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 108/627 objects degraded (17.225%) 2024-04-22T15:40:37.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:36 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:37.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:36 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:37.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:36 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:40:37.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:36 smithi145 ceph-mon[82523]: pgmap v190: 161 pgs: 27 active+undersized+degraded, 41 active+undersized, 93 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 108/627 objects degraded (17.225%) 2024-04-22T15:40:37.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:36 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:37.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:36 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:37.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:36 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:40:37.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:36 smithi145 ceph-mon[88162]: pgmap v190: 161 pgs: 27 active+undersized+degraded, 41 active+undersized, 93 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 108/627 objects degraded (17.225%) 2024-04-22T15:40:37.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:36 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:37.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:36 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:37.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:36 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:40:39.148 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:39 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4[76239]: 2024-04-22T15:40:39.068+0000 7ff5c3e187c0 -1 osd.4 0 read_superblock omap replica is missing. 2024-04-22T15:40:39.148 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:39 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4[76239]: 2024-04-22T15:40:39.098+0000 7ff5c3e187c0 -1 osd.4 128 log_to_monitors true 2024-04-22T15:40:39.148 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:38 smithi172 ceph-mon[65307]: pgmap v191: 161 pgs: 27 active+undersized+degraded, 41 active+undersized, 93 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 108/627 objects degraded (17.225%) 2024-04-22T15:40:39.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:38 smithi145 ceph-mon[82523]: pgmap v191: 161 pgs: 27 active+undersized+degraded, 41 active+undersized, 93 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 108/627 objects degraded (17.225%) 2024-04-22T15:40:39.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:38 smithi145 ceph-mon[88162]: pgmap v191: 161 pgs: 27 active+undersized+degraded, 41 active+undersized, 93 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 108/627 objects degraded (17.225%) 2024-04-22T15:40:40.217 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:40:40 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4[76239]: 2024-04-22T15:40:40.005+0000 7ff5b5270700 -1 osd.4 128 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T15:40:40.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:39 smithi172 ceph-mon[65307]: from='osd.4 [v2:172.21.15.172:6800/836348173,v1:172.21.15.172:6801/836348173]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T15:40:40.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:39 smithi172 ceph-mon[65307]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T15:40:40.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:39 smithi172 ceph-mon[65307]: Health check update: Degraded data redundancy: 108/627 objects degraded (17.225%), 27 pgs degraded (PG_DEGRADED) 2024-04-22T15:40:40.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:39 smithi145 ceph-mon[82523]: from='osd.4 [v2:172.21.15.172:6800/836348173,v1:172.21.15.172:6801/836348173]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T15:40:40.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:39 smithi145 ceph-mon[82523]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T15:40:40.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:39 smithi145 ceph-mon[82523]: Health check update: Degraded data redundancy: 108/627 objects degraded (17.225%), 27 pgs degraded (PG_DEGRADED) 2024-04-22T15:40:40.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:39 smithi145 ceph-mon[88162]: from='osd.4 [v2:172.21.15.172:6800/836348173,v1:172.21.15.172:6801/836348173]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T15:40:40.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:39 smithi145 ceph-mon[88162]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T15:40:40.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:39 smithi145 ceph-mon[88162]: Health check update: Degraded data redundancy: 108/627 objects degraded (17.225%), 27 pgs degraded (PG_DEGRADED) 2024-04-22T15:40:41.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:40 smithi145 ceph-mon[88162]: pgmap v192: 161 pgs: 27 active+undersized+degraded, 41 active+undersized, 93 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 108/627 objects degraded (17.225%) 2024-04-22T15:40:41.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:40 smithi145 ceph-mon[88162]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-22T15:40:41.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:40 smithi145 ceph-mon[88162]: osdmap e131: 8 total, 7 up, 8 in 2024-04-22T15:40:41.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:40 smithi145 ceph-mon[88162]: from='osd.4 [v2:172.21.15.172:6800/836348173,v1:172.21.15.172:6801/836348173]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:40:41.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:40 smithi145 ceph-mon[88162]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:40:41.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:40 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:41.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:40 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:41.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:40 smithi145 ceph-mon[82523]: pgmap v192: 161 pgs: 27 active+undersized+degraded, 41 active+undersized, 93 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 108/627 objects degraded (17.225%) 2024-04-22T15:40:41.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:40 smithi145 ceph-mon[82523]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-22T15:40:41.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:40 smithi145 ceph-mon[82523]: osdmap e131: 8 total, 7 up, 8 in 2024-04-22T15:40:41.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:40 smithi145 ceph-mon[82523]: from='osd.4 [v2:172.21.15.172:6800/836348173,v1:172.21.15.172:6801/836348173]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:40:41.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:40 smithi145 ceph-mon[82523]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:40:41.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:40 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:41.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:40 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:41.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:40 smithi172 ceph-mon[65307]: pgmap v192: 161 pgs: 27 active+undersized+degraded, 41 active+undersized, 93 active+clean; 585 KiB data, 170 MiB used, 715 GiB / 715 GiB avail; 108/627 objects degraded (17.225%) 2024-04-22T15:40:41.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:40 smithi172 ceph-mon[65307]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-22T15:40:41.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:40 smithi172 ceph-mon[65307]: osdmap e131: 8 total, 7 up, 8 in 2024-04-22T15:40:41.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:40 smithi172 ceph-mon[65307]: from='osd.4 [v2:172.21.15.172:6800/836348173,v1:172.21.15.172:6801/836348173]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:40:41.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:40 smithi172 ceph-mon[65307]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:40:41.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:40 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:41.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:40 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:42.047 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:41 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:40:42.047 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:41 smithi172 ceph-mon[65307]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:40:42.047 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:41 smithi172 ceph-mon[65307]: osd.4 [v2:172.21.15.172:6800/836348173,v1:172.21.15.172:6801/836348173] boot 2024-04-22T15:40:42.047 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:41 smithi172 ceph-mon[65307]: osdmap e132: 8 total, 8 up, 8 in 2024-04-22T15:40:42.048 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:41 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:40:42.048 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:41 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:42.048 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:41 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:42.048 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:41 smithi172 ceph-mon[65307]: osdmap e133: 8 total, 8 up, 8 in 2024-04-22T15:40:42.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:41 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:40:42.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:41 smithi145 ceph-mon[82523]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:40:42.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:41 smithi145 ceph-mon[82523]: osd.4 [v2:172.21.15.172:6800/836348173,v1:172.21.15.172:6801/836348173] boot 2024-04-22T15:40:42.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:41 smithi145 ceph-mon[82523]: osdmap e132: 8 total, 8 up, 8 in 2024-04-22T15:40:42.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:41 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:40:42.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:41 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:42.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:41 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:42.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:41 smithi145 ceph-mon[82523]: osdmap e133: 8 total, 8 up, 8 in 2024-04-22T15:40:42.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:41 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:40:42.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:41 smithi145 ceph-mon[88162]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:40:42.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:41 smithi145 ceph-mon[88162]: osd.4 [v2:172.21.15.172:6800/836348173,v1:172.21.15.172:6801/836348173] boot 2024-04-22T15:40:42.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:41 smithi145 ceph-mon[88162]: osdmap e132: 8 total, 8 up, 8 in 2024-04-22T15:40:42.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:41 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T15:40:42.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:41 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:42.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:41 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:42.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:41 smithi145 ceph-mon[88162]: osdmap e133: 8 total, 8 up, 8 in 2024-04-22T15:40:43.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:43 smithi172 ceph-mon[65307]: pgmap v195: 161 pgs: 27 active+undersized+degraded, 41 active+undersized, 93 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 108/627 objects degraded (17.225%) 2024-04-22T15:40:43.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:43 smithi145 ceph-mon[82523]: pgmap v195: 161 pgs: 27 active+undersized+degraded, 41 active+undersized, 93 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 108/627 objects degraded (17.225%) 2024-04-22T15:40:43.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:43 smithi145 ceph-mon[88162]: pgmap v195: 161 pgs: 27 active+undersized+degraded, 41 active+undersized, 93 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 108/627 objects degraded (17.225%) 2024-04-22T15:40:45.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:40:45] "GET /metrics HTTP/1.1" 200 38200 "" "Prometheus/2.43.0" 2024-04-22T15:40:45.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:45 smithi145 ceph-mon[82523]: pgmap v197: 161 pgs: 21 active+undersized+degraded, 28 active+undersized, 112 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 81/627 objects degraded (12.919%) 2024-04-22T15:40:45.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:45 smithi145 ceph-mon[82523]: Health check update: Degraded data redundancy: 81/627 objects degraded (12.919%), 21 pgs degraded (PG_DEGRADED) 2024-04-22T15:40:45.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:45 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:45.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:45 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:45.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:45 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:40:45.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:45 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:40:45.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:45 smithi145 ceph-mon[82523]: pgmap v198: 161 pgs: 21 active+undersized+degraded, 28 active+undersized, 112 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 81/627 objects degraded (12.919%) 2024-04-22T15:40:45.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:45 smithi145 ceph-mon[82523]: pgmap v199: 161 pgs: 21 active+undersized+degraded, 28 active+undersized, 112 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 81/627 objects degraded (12.919%) 2024-04-22T15:40:45.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:45 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:45.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:45 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:40:45.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:45 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:40:45.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:45 smithi145 ceph-mon[88162]: pgmap v197: 161 pgs: 21 active+undersized+degraded, 28 active+undersized, 112 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 81/627 objects degraded (12.919%) 2024-04-22T15:40:45.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:45 smithi145 ceph-mon[88162]: Health check update: Degraded data redundancy: 81/627 objects degraded (12.919%), 21 pgs degraded (PG_DEGRADED) 2024-04-22T15:40:45.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:45 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:45.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:45 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:45.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:45 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:40:45.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:45 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:40:45.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:45 smithi145 ceph-mon[88162]: pgmap v198: 161 pgs: 21 active+undersized+degraded, 28 active+undersized, 112 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 81/627 objects degraded (12.919%) 2024-04-22T15:40:45.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:45 smithi145 ceph-mon[88162]: pgmap v199: 161 pgs: 21 active+undersized+degraded, 28 active+undersized, 112 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 81/627 objects degraded (12.919%) 2024-04-22T15:40:45.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:45 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:45.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:45 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:40:45.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:45 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:40:45.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:45 smithi172 ceph-mon[65307]: pgmap v197: 161 pgs: 21 active+undersized+degraded, 28 active+undersized, 112 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 81/627 objects degraded (12.919%) 2024-04-22T15:40:45.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:45 smithi172 ceph-mon[65307]: Health check update: Degraded data redundancy: 81/627 objects degraded (12.919%), 21 pgs degraded (PG_DEGRADED) 2024-04-22T15:40:45.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:45 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:45.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:45 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:45.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:45 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:40:45.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:45 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:40:45.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:45 smithi172 ceph-mon[65307]: pgmap v198: 161 pgs: 21 active+undersized+degraded, 28 active+undersized, 112 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 81/627 objects degraded (12.919%) 2024-04-22T15:40:45.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:45 smithi172 ceph-mon[65307]: pgmap v199: 161 pgs: 21 active+undersized+degraded, 28 active+undersized, 112 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 81/627 objects degraded (12.919%) 2024-04-22T15:40:45.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:45 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:45.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:45 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:40:45.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:45 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:40:46.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-mon[82523]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:40:46.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-mon[82523]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:40:46.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:40:46.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:46.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:46.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:46.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T15:40:46.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-mon[88162]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:40:46.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-mon[88162]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:40:46.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:40:46.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:46.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:46.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:46.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T15:40:46.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:40:46.270+0000 7f17b18ae700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:40:46.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:40:46.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:40:46.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:40:46.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:40:46.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:46.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:46.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:40:46.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:40:46.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:40:46.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:40:46.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:40:46.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:40:46.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:40:46.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:40:46.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:40:46.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:40:46.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:40:46.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:40:46.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:46.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:46.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:40:46.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:40:46.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:40:46.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:40:46.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:40:46.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:40:46.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:40:46.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:40:46.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:40:46.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:40:46.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:40:46.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:40:46.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:40:46.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:40:46.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:40:46.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:40:46.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:40:46.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:40:46.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:40:46.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:40:46.378 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:40:46.378 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:40:46.378 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:46.378 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:46.378 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:40:46.378 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:40:46.378 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:40:46.378 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:40:46.378 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:40:46.378 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:40:46.378 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:40:46.378 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:40:46.378 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:40:46.379 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:40:46.379 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:40:46.379 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:40:46.379 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:46.379 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:46.379 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:46 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:40:46.282+0000 7f17e84c2700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-04-22T15:40:46.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:46 smithi172 ceph-mon[65307]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:40:46.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:46 smithi172 ceph-mon[65307]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:40:46.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:46 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:40:46.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:46 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:46.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:46 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:46.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:46 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:40:46.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:46 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T15:40:47.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:47 smithi145 ceph-mon[82523]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:40:47.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:40:47.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:40:47.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:40:47.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:40:47.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:47.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:47.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:40:47.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:40:47.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:40:47.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:40:47.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:40:47.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:40:47.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:40:47.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:40:47.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:40:47.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:40:47.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:40:47.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:40:47.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:47.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:47.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:40:47.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:40:47.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:40:47.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:40:47.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:40:47.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:40:47.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:40:47.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:40:47.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:40:47.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:40:47.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:40:47.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:40:47.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:40:47.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:40:47.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:40:47.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:40:47.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:40:47.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:40:47.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:40:47.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:40:47.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:40:47.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:40:47.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:47.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:47.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:40:47.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:40:47.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:40:47.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:40:47.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:40:47.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:40:47.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:40:47.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:40:47.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:40:47.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:47 smithi145 ceph-mon[88162]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:40:47.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:40:47.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:40:47.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:40:47.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:40:47.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:47.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:47.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:40:47.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:40:47.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:40:47.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:40:47.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:40:47.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:40:47.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:40:47.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:40:47.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:40:47.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:40:47.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:40:47.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:40:47.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:47.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:47.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:40:47.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:40:47.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:40:47.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:40:47.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:40:47.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:40:47.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:40:47.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:40:47.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:40:47.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:40:47.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:40:47.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:40:47.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:40:47.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:40:47.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:40:47.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:40:47.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:40:47.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:40:47.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:40:47.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:40:47.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:40:47.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:40:47.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:47.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:47.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:40:47.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:40:47.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:40:47.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:40:47.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:40:47.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:40:47.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:40:47.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:40:47.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:40:47.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:40:47.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:40:47.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:40:47.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:47.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:47.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:47 smithi145 ceph-mon[82523]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T15:40:47.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:47 smithi145 ceph-mon[82523]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-04-22T15:40:47.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:47 smithi145 ceph-mon[82523]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 959 B/s rd, 0 op/s 2024-04-22T15:40:47.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:40:47.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:40:47.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:40:47.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:47.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:47.632 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:47 smithi145 ceph-mon[88162]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T15:40:47.632 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:47 smithi145 ceph-mon[88162]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-04-22T15:40:47.632 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:47 smithi145 ceph-mon[88162]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 959 B/s rd, 0 op/s 2024-04-22T15:40:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:47 smithi172 ceph-mon[65307]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:40:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:40:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:40:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:40:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:40:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:40:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:40:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:40:47.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:40:47.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:40:47.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:40:47.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:40:47.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:40:47.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:40:47.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:40:47.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:40:47.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:40:47.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:47.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:47.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:40:47.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:40:47.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:40:47.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:40:47.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:40:47.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:40:47.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:40:47.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:40:47.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:40:47.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:40:47.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:40:47.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:40:47.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:40:47.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:40:47.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:40:47.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:40:47.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:40:47.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:40:47.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:40:47.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:40:47.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:40:47.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:40:47.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:47.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:47.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:40:47.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:40:47.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:40:47.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:40:47.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:40:47.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:40:47.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:40:47.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:40:47.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:40:47.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:40:47.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:40:47.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:40:47.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:40:47.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:40:47.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:47 smithi172 ceph-mon[65307]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T15:40:47.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:47 smithi172 ceph-mon[65307]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-04-22T15:40:47.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:47 smithi172 ceph-mon[65307]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 959 B/s rd, 0 op/s 2024-04-22T15:40:48.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:48 smithi145 ceph-mon[82523]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 81/627 objects degraded (12.919%), 21 pgs degraded) 2024-04-22T15:40:48.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:48 smithi145 ceph-mon[88162]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 81/627 objects degraded (12.919%), 21 pgs degraded) 2024-04-22T15:40:48.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:48 smithi172 ceph-mon[65307]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 81/627 objects degraded (12.919%), 21 pgs degraded) 2024-04-22T15:40:49.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:49 smithi145 ceph-mon[82523]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-22T15:40:49.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:49 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:49.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:49 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:40:49.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:49 smithi145 ceph-mon[88162]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-22T15:40:49.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:49 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:49.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:49 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:40:49.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:49 smithi172 ceph-mon[65307]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-22T15:40:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:49 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:49.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:49 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:40:51.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:51 smithi145 ceph-mon[88162]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:40:51.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:51 smithi145 ceph-mon[82523]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:40:51.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:51 smithi172 ceph-mon[65307]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:40:52.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:52 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:40:52.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:52 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:40:52.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:52 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:40:53.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:53 smithi145 ceph-mon[82523]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 606 B/s rd, 0 op/s 2024-04-22T15:40:53.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:53 smithi145 ceph-mon[88162]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 606 B/s rd, 0 op/s 2024-04-22T15:40:53.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:53 smithi172 ceph-mon[65307]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 606 B/s rd, 0 op/s 2024-04-22T15:40:55.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:55 smithi145 ceph-mon[82523]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:40:55.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:55 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:55.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:40:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:40:55] "GET /metrics HTTP/1.1" 200 38209 "" "Prometheus/2.43.0" 2024-04-22T15:40:55.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:55 smithi145 ceph-mon[88162]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:40:55.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:55 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:55.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:55 smithi172 ceph-mon[65307]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:40:55.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:55 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:40:57.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:57 smithi145 ceph-mon[82523]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:40:57.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:57 smithi145 ceph-mon[88162]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:40:57.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:57 smithi172 ceph-mon[65307]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:40:59.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:40:59 smithi145 ceph-mon[82523]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:40:59.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:40:59 smithi145 ceph-mon[88162]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:40:59.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:40:59 smithi172 ceph-mon[65307]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:41:01.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:01 smithi145 ceph-mon[82523]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:41:01.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:01 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T15:41:01.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:01 smithi145 ceph-mon[88162]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:41:01.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:01 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T15:41:01.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:01 smithi172 ceph-mon[65307]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:41:01.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:01 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T15:41:02.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:02 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:41:02.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:02 smithi172 ceph-mon[65307]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T15:41:02.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:02 smithi172 ceph-mon[65307]: Upgrade: osd.5 is safe to restart 2024-04-22T15:41:02.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:02 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:41:02.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:02 smithi145 ceph-mon[88162]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T15:41:02.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:02 smithi145 ceph-mon[88162]: Upgrade: osd.5 is safe to restart 2024-04-22T15:41:02.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:02 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:41:02.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:02 smithi145 ceph-mon[82523]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-22T15:41:02.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:02 smithi145 ceph-mon[82523]: Upgrade: osd.5 is safe to restart 2024-04-22T15:41:03.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:03 smithi145 ceph-mon[88162]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:41:03.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:03 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:03.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:03 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T15:41:03.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:03 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:41:03.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:03 smithi145 ceph-mon[82523]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:41:03.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:03 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:03.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:03 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T15:41:03.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:03 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:41:03.651 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:03 smithi172 ceph-mon[65307]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:41:03.651 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:03 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:03.651 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:03 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T15:41:03.651 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:03 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:41:03.718 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:41:04.373 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:41:04.373 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (7m) 45s ago 30m 27.0M - 0.25.0 c8568f914cd2 1c58ccf37ffd 2024-04-22T15:41:04.373 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (7m) 24s ago 29m 79.8M - 9.4.12 2bacad6d85d8 2f825d15ab10 2024-04-22T15:41:04.373 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 unknown 45s ago 29m 60.7M - 3.5 e1d6a67b021e 9704631ffa0e 2024-04-22T15:41:04.373 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443,9283,8765 running (9m) 24s ago 34m 453M - 19.0.0-3223-g5883d494 3d983c26e278 e7500ef71604 2024-04-22T15:41:04.373 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (19m) 45s ago 36m 511M - 19.0.0-3223-g5883d494 3d983c26e278 b2de97566cf4 2024-04-22T15:41:04.373 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (5m) 45s ago 36m 53.7M 2048M 19.0.0-3223-g5883d494 3d983c26e278 7bfdc11120f4 2024-04-22T15:41:04.374 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (6m) 24s ago 35m 49.6M 2048M 19.0.0-3223-g5883d494 3d983c26e278 0a360b08a6d6 2024-04-22T15:41:04.374 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (4m) 45s ago 35m 42.0M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38dcfa677c18 2024-04-22T15:41:04.374 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (8m) 45s ago 30m 15.3M - 1.5.0 0da6a335fe13 8c75c26fec05 2024-04-22T15:41:04.374 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (8m) 24s ago 30m 15.9M - 1.5.0 0da6a335fe13 8fd2949e2eaf 2024-04-22T15:41:04.374 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (3m) 45s ago 34m 51.7M 2992M 19.0.0-3223-g5883d494 3d983c26e278 e4bc06317af0 2024-04-22T15:41:04.374 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (2m) 45s ago 34m 44.0M 2992M 19.0.0-3223-g5883d494 3d983c26e278 d299f622f08d 2024-04-22T15:41:04.374 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (109s) 45s ago 33m 41.7M 2992M 19.0.0-3223-g5883d494 3d983c26e278 6aea4a926fe4 2024-04-22T15:41:04.374 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (50s) 45s ago 33m 15.1M 2992M 19.0.0-3223-g5883d494 3d983c26e278 c0cd2bdc1439 2024-04-22T15:41:04.374 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (28s) 24s ago 32m 12.5M 3248M 19.0.0-3223-g5883d494 3d983c26e278 7ac5776bc07a 2024-04-22T15:41:04.374 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (32m) 24s ago 32m 57.6M 3248M 17.2.0 e1d6a67b021e ec3ab1335b5c 2024-04-22T15:41:04.374 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (32m) 24s ago 32m 58.3M 3248M 17.2.0 e1d6a67b021e 45d475edf1ca 2024-04-22T15:41:04.374 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (31m) 24s ago 31m 58.7M 3248M 17.2.0 e1d6a67b021e cc9e1d9e7916 2024-04-22T15:41:04.374 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (8m) 24s ago 31m 70.5M - 2.43.0 a07b618ecd1d 51cf4df6e8d4 2024-04-22T15:41:04.375 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (29m) 45s ago 29m 80.2M - 17.2.0 e1d6a67b021e c90a636c947e 2024-04-22T15:41:04.375 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (29m) 24s ago 29m 77.9M - 17.2.0 e1d6a67b021e 20ac46e9bfda 2024-04-22T15:41:04.595 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:04 smithi172 ceph-mon[65307]: Upgrade: Updating osd.5 2024-04-22T15:41:04.595 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:04 smithi172 ceph-mon[65307]: Deploying daemon osd.5 on smithi172 2024-04-22T15:41:04.595 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:04 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:41:04.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:04 smithi145 ceph-mon[88162]: Upgrade: Updating osd.5 2024-04-22T15:41:04.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:04 smithi145 ceph-mon[88162]: Deploying daemon osd.5 on smithi172 2024-04-22T15:41:04.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:04 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:41:04.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:04 smithi145 ceph-mon[82523]: Upgrade: Updating osd.5 2024-04-22T15:41:04.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:04 smithi145 ceph-mon[82523]: Deploying daemon osd.5 on smithi172 2024-04-22T15:41:04.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:04 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:41:04.773 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:41:04.773 INFO:teuthology.orchestra.run.smithi145.stdout: "mon": { 2024-04-22T15:41:04.773 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 3 2024-04-22T15:41:04.773 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:41:04.773 INFO:teuthology.orchestra.run.smithi145.stdout: "mgr": { 2024-04-22T15:41:04.773 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:41:04.773 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:41:04.774 INFO:teuthology.orchestra.run.smithi145.stdout: "osd": { 2024-04-22T15:41:04.774 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-04-22T15:41:04.774 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 5 2024-04-22T15:41:04.774 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:41:04.774 INFO:teuthology.orchestra.run.smithi145.stdout: "rgw": { 2024-04-22T15:41:04.774 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:41:04.774 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:41:04.774 INFO:teuthology.orchestra.run.smithi145.stdout: "overall": { 2024-04-22T15:41:04.774 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-04-22T15:41:04.774 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 10 2024-04-22T15:41:04.774 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:41:04.774 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:41:05.105 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:41:05.105 INFO:teuthology.orchestra.run.smithi145.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:41:05.105 INFO:teuthology.orchestra.run.smithi145.stdout: "in_progress": true, 2024-04-22T15:41:05.105 INFO:teuthology.orchestra.run.smithi145.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-04-22T15:41:05.105 INFO:teuthology.orchestra.run.smithi145.stdout: "services_complete": [], 2024-04-22T15:41:05.106 INFO:teuthology.orchestra.run.smithi145.stdout: "progress": "5/8 daemons upgraded", 2024-04-22T15:41:05.106 INFO:teuthology.orchestra.run.smithi145.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T15:41:05.106 INFO:teuthology.orchestra.run.smithi145.stdout: "is_paused": false 2024-04-22T15:41:05.106 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:41:05.360 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:05 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:41:05] "GET /metrics HTTP/1.1" 200 38208 "" "Prometheus/2.43.0" 2024-04-22T15:41:05.546 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:05 smithi172 ceph-mon[65307]: from='client.44505 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:41:05.549 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:05 smithi172 ceph-mon[65307]: from='client.54448 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:41:05.549 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:05 smithi172 ceph-mon[65307]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:41:05.549 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:05 smithi172 ceph-mon[65307]: from='client.54454 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:41:05.549 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:05 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/1889339330' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:41:05.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:05 smithi145 ceph-mon[82523]: from='client.44505 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:41:05.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:05 smithi145 ceph-mon[82523]: from='client.54448 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:41:05.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:05 smithi145 ceph-mon[82523]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:41:05.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:05 smithi145 ceph-mon[82523]: from='client.54454 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:41:05.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:05 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/1889339330' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:41:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:05 smithi145 ceph-mon[88162]: from='client.44505 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:41:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:05 smithi145 ceph-mon[88162]: from='client.54448 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:41:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:05 smithi145 ceph-mon[88162]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:41:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:05 smithi145 ceph-mon[88162]: from='client.54454 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:41:05.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:05 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/1889339330' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:41:06.152 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:05 smithi172 systemd[1]: Stopping Ceph osd.5 for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:41:06.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:06 smithi172 ceph-mon[65307]: from='client.54466 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:41:06.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:06 smithi172 ceph-mon[65307]: osd.5 marked itself down and dead 2024-04-22T15:41:06.468 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:06 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[29081]: 2024-04-22T15:41:06.147+0000 7fe5961a8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:41:06.468 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:06 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[29081]: 2024-04-22T15:41:06.147+0000 7fe5961a8700 -1 osd.5 133 *** Got signal Terminated *** 2024-04-22T15:41:06.468 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:06 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[29081]: 2024-04-22T15:41:06.147+0000 7fe5961a8700 -1 osd.5 133 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T15:41:06.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:06 smithi145 ceph-mon[88162]: from='client.54466 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:41:06.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:06 smithi145 ceph-mon[88162]: osd.5 marked itself down and dead 2024-04-22T15:41:06.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:06 smithi145 ceph-mon[82523]: from='client.54466 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:41:06.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:06 smithi145 ceph-mon[82523]: osd.5 marked itself down and dead 2024-04-22T15:41:07.019 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:06 smithi172 podman[80454]: 2024-04-22 15:41:06.729366836 +0000 UTC m=+0.682001716 container died ec3ab1335b5cfef79b3920149290f82c7048aca5c53bcef87572f6a0a6f13811 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, vcs-type=git, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, ceph=True) 2024-04-22T15:41:07.368 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:07 smithi172 podman[80454]: 2024-04-22 15:41:07.115549891 +0000 UTC m=+1.068184771 container cleanup ec3ab1335b5cfef79b3920149290f82c7048aca5c53bcef87572f6a0a6f13811 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , distribution-scope=public, RELEASE=HEAD, vcs-type=git, com.redhat.component=centos-stream-container, release=754, name=centos-stream, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-22T15:41:07.368 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:07 smithi172 bash[80454]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5 2024-04-22T15:41:07.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:07 smithi145 ceph-mon[82523]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:41:07.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:07 smithi145 ceph-mon[82523]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:41:07.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:07 smithi145 ceph-mon[82523]: osdmap e134: 8 total, 7 up, 8 in 2024-04-22T15:41:07.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:07 smithi145 ceph-mon[88162]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:41:07.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:07 smithi145 ceph-mon[88162]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:41:07.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:07 smithi145 ceph-mon[88162]: osdmap e134: 8 total, 7 up, 8 in 2024-04-22T15:41:07.717 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:07 smithi172 podman[80483]: 2024-04-22 15:41:07.369000319 +0000 UTC m=+0.642884533 container remove ec3ab1335b5cfef79b3920149290f82c7048aca5c53bcef87572f6a0a6f13811 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, distribution-scope=public, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, vcs-type=git, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , RELEASE=HEAD, vendor=Red Hat, Inc., architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container) 2024-04-22T15:41:07.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:07 smithi172 ceph-mon[65307]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:41:07.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:07 smithi172 ceph-mon[65307]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:41:07.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:07 smithi172 ceph-mon[65307]: osdmap e134: 8 total, 7 up, 8 in 2024-04-22T15:41:08.041 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:07 smithi172 podman[80578]: 2024-04-22 15:41:07.632140432 +0000 UTC m=+0.018296254 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:41:08.042 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:07 smithi172 podman[80578]: 2024-04-22 15:41:07.736791387 +0000 UTC m=+0.122947237 container create b640a8dac1b7da9f54e640bf3abcb9775d1bf3a93065d8fc952f56e6143e7c49 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_CLEAN=True, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-04-22T15:41:08.042 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:07 smithi172 podman[80578]: 2024-04-22 15:41:07.964325566 +0000 UTC m=+0.350481373 container init b640a8dac1b7da9f54e640bf3abcb9775d1bf3a93065d8fc952f56e6143e7c49 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-04-22T15:41:08.042 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:07 smithi172 podman[80578]: 2024-04-22 15:41:07.967565062 +0000 UTC m=+0.353720873 container start b640a8dac1b7da9f54e640bf3abcb9775d1bf3a93065d8fc952f56e6143e7c49 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:41:08.042 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:08 smithi172 podman[80578]: 2024-04-22 15:41:08.039413567 +0000 UTC m=+0.425569376 container attach b640a8dac1b7da9f54e640bf3abcb9775d1bf3a93065d8fc952f56e6143e7c49 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T15:41:08.380 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:08 smithi172 podman[80578]: 2024-04-22 15:41:08.256264295 +0000 UTC m=+0.642420106 container died b640a8dac1b7da9f54e640bf3abcb9775d1bf3a93065d8fc952f56e6143e7c49 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:41:08.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:08 smithi172 ceph-mon[65307]: osdmap e135: 8 total, 7 up, 8 in 2024-04-22T15:41:08.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:08 smithi145 ceph-mon[82523]: osdmap e135: 8 total, 7 up, 8 in 2024-04-22T15:41:08.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:08 smithi145 ceph-mon[88162]: osdmap e135: 8 total, 7 up, 8 in 2024-04-22T15:41:09.147 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:08 smithi172 podman[80604]: 2024-04-22 15:41:08.927404209 +0000 UTC m=+0.662466043 container remove b640a8dac1b7da9f54e640bf3abcb9775d1bf3a93065d8fc952f56e6143e7c49 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T15:41:09.147 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:08 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.5.service: Deactivated successfully. 2024-04-22T15:41:09.147 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:08 smithi172 systemd[1]: Stopped Ceph osd.5 for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:41:09.147 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:08 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.5.service: Consumed 16.290s CPU time. 2024-04-22T15:41:09.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:09 smithi172 ceph-mon[65307]: pgmap v213: 161 pgs: 9 active+undersized+degraded, 11 stale+active+clean, 20 active+undersized, 121 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 38/627 objects degraded (6.061%) 2024-04-22T15:41:09.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:09 smithi172 ceph-mon[65307]: Health check failed: Degraded data redundancy: 38/627 objects degraded (6.061%), 9 pgs degraded (PG_DEGRADED) 2024-04-22T15:41:09.468 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:09 smithi172 systemd[1]: Starting Ceph osd.5 for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:41:09.468 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:09 smithi172 podman[80709]: 2024-04-22 15:41:09.266655218 +0000 UTC m=+0.018353415 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:41:09.468 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:09 smithi172 podman[80709]: 2024-04-22 15:41:09.370613668 +0000 UTC m=+0.122311849 container create f76752ef1a33b6f01664a495b29b5eef1a59943102c1f63ab880d4e42f8af58d (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5-activate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux ) 2024-04-22T15:41:09.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:09 smithi145 ceph-mon[82523]: pgmap v213: 161 pgs: 9 active+undersized+degraded, 11 stale+active+clean, 20 active+undersized, 121 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 38/627 objects degraded (6.061%) 2024-04-22T15:41:09.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:09 smithi145 ceph-mon[82523]: Health check failed: Degraded data redundancy: 38/627 objects degraded (6.061%), 9 pgs degraded (PG_DEGRADED) 2024-04-22T15:41:09.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:09 smithi145 ceph-mon[88162]: pgmap v213: 161 pgs: 9 active+undersized+degraded, 11 stale+active+clean, 20 active+undersized, 121 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 38/627 objects degraded (6.061%) 2024-04-22T15:41:09.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:09 smithi145 ceph-mon[88162]: Health check failed: Degraded data redundancy: 38/627 objects degraded (6.061%), 9 pgs degraded (PG_DEGRADED) 2024-04-22T15:41:09.967 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:09 smithi172 podman[80709]: 2024-04-22 15:41:09.581341221 +0000 UTC m=+0.333039407 container init f76752ef1a33b6f01664a495b29b5eef1a59943102c1f63ab880d4e42f8af58d (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5-activate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-04-22T15:41:09.968 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:09 smithi172 podman[80709]: 2024-04-22 15:41:09.584478851 +0000 UTC m=+0.336177035 container start f76752ef1a33b6f01664a495b29b5eef1a59943102c1f63ab880d4e42f8af58d (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5-activate, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:41:09.968 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:09 smithi172 podman[80709]: 2024-04-22 15:41:09.656222641 +0000 UTC m=+0.407920829 container attach f76752ef1a33b6f01664a495b29b5eef1a59943102c1f63ab880d4e42f8af58d (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5-activate, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True) 2024-04-22T15:41:10.371 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:10 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5-activate[80724]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:41:10.375 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:10 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5-activate[80724]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:41:10.375 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:10 smithi172 bash[80709]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:41:10.375 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:10 smithi172 bash[80709]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:41:10.375 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:10 smithi172 bash[80709]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-22T15:41:10.375 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:10 smithi172 bash[80709]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-22T15:41:10.375 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:10 smithi172 bash[80709]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-22T15:41:10.375 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:10 smithi172 bash[80709]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-22T15:41:10.375 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:10 smithi172 bash[80709]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-04-22T15:41:10.376 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:10 smithi172 bash[80709]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-22T15:41:10.376 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:10 smithi172 bash[80709]: --> ceph-volume raw activate successful for osd ID: 5 2024-04-22T15:41:10.376 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:10 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5-activate[80724]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-22T15:41:10.376 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:10 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5-activate[80724]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-22T15:41:10.376 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:10 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5-activate[80724]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-22T15:41:10.376 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:10 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5-activate[80724]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-22T15:41:10.376 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:10 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5-activate[80724]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-04-22T15:41:10.376 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:10 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5-activate[80724]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-22T15:41:10.376 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:10 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5-activate[80724]: --> ceph-volume raw activate successful for osd ID: 5 2024-04-22T15:41:10.376 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:10 smithi172 podman[80709]: 2024-04-22 15:41:10.084020352 +0000 UTC m=+0.835718542 container died f76752ef1a33b6f01664a495b29b5eef1a59943102c1f63ab880d4e42f8af58d (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5-activate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-04-22T15:41:10.672 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:10 smithi172 ceph-mon[65307]: pgmap v214: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 71/627 objects degraded (11.324%) 2024-04-22T15:41:10.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:10 smithi145 ceph-mon[88162]: pgmap v214: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 71/627 objects degraded (11.324%) 2024-04-22T15:41:10.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:10 smithi145 ceph-mon[82523]: pgmap v214: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 71/627 objects degraded (11.324%) 2024-04-22T15:41:10.967 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:10 smithi172 podman[80862]: 2024-04-22 15:41:10.669023507 +0000 UTC m=+0.579821683 container remove f76752ef1a33b6f01664a495b29b5eef1a59943102c1f63ab880d4e42f8af58d (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5-activate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-04-22T15:41:10.967 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:10 smithi172 podman[80925]: 2024-04-22 15:41:10.861541014 +0000 UTC m=+0.092788514 container create 5f803668a17b6fdff528ee806b9e6ff89fbb65b4877b300b5c329905f57179b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-04-22T15:41:10.967 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:10 smithi172 podman[80925]: 2024-04-22 15:41:10.786993016 +0000 UTC m=+0.018240527 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:41:11.468 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:11 smithi172 podman[80925]: 2024-04-22 15:41:11.089458241 +0000 UTC m=+0.320705749 container init 5f803668a17b6fdff528ee806b9e6ff89fbb65b4877b300b5c329905f57179b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T15:41:11.468 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:11 smithi172 podman[80925]: 2024-04-22 15:41:11.092642016 +0000 UTC m=+0.323889516 container start 5f803668a17b6fdff528ee806b9e6ff89fbb65b4877b300b5c329905f57179b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=) 2024-04-22T15:41:11.468 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:11 smithi172 ceph-osd[80944]: -- 172.21.15.172:0/3264351512 <== mon.2 v2:172.21.15.172:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5598dd293680 con 0x5598dd25d400 2024-04-22T15:41:11.468 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:11 smithi172 bash[80925]: 5f803668a17b6fdff528ee806b9e6ff89fbb65b4877b300b5c329905f57179b1 2024-04-22T15:41:11.468 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:11 smithi172 systemd[1]: Started Ceph osd.5 for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:41:11.967 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:11 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:41:11.631+0000 7fc59684c7c0 -1 Falling back to public interface 2024-04-22T15:41:12.310 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:12 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:41:12.311 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:12 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:12.311 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:12 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:12.311 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:12 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:41:12.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:12 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:41:12.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:12 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:12.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:12 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:12.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:12 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:41:12.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:12 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:41:12.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:12 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:12.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:12 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:12.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:12 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:41:13.205 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:13 smithi172 ceph-mon[65307]: pgmap v215: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 71/627 objects degraded (11.324%) 2024-04-22T15:41:13.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:13 smithi145 ceph-mon[88162]: pgmap v215: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 71/627 objects degraded (11.324%) 2024-04-22T15:41:13.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:13 smithi145 ceph-mon[82523]: pgmap v215: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 71/627 objects degraded (11.324%) 2024-04-22T15:41:14.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:14 smithi145 ceph-mon[82523]: Health check update: Degraded data redundancy: 71/627 objects degraded (11.324%), 20 pgs degraded (PG_DEGRADED) 2024-04-22T15:41:14.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:14 smithi145 ceph-mon[88162]: Health check update: Degraded data redundancy: 71/627 objects degraded (11.324%), 20 pgs degraded (PG_DEGRADED) 2024-04-22T15:41:14.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:14 smithi172 ceph-mon[65307]: Health check update: Degraded data redundancy: 71/627 objects degraded (11.324%), 20 pgs degraded (PG_DEGRADED) 2024-04-22T15:41:14.718 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:14 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:41:14.334+0000 7fc59684c7c0 -1 osd.5 0 read_superblock omap replica is missing. 2024-04-22T15:41:14.718 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:14 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:41:14.368+0000 7fc59684c7c0 -1 osd.5 133 log_to_monitors true 2024-04-22T15:41:15.622 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:41:15] "GET /metrics HTTP/1.1" 200 38208 "" "Prometheus/2.43.0" 2024-04-22T15:41:15.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:15 smithi145 ceph-mon[82523]: pgmap v216: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 71/627 objects degraded (11.324%) 2024-04-22T15:41:15.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:15 smithi145 ceph-mon[82523]: from='osd.5 [v2:172.21.15.172:6808/3430112748,v1:172.21.15.172:6809/3430112748]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T15:41:15.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:15 smithi145 ceph-mon[82523]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T15:41:15.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:15 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:15.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:15 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:15.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:15 smithi145 ceph-mon[88162]: pgmap v216: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 71/627 objects degraded (11.324%) 2024-04-22T15:41:15.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:15 smithi145 ceph-mon[88162]: from='osd.5 [v2:172.21.15.172:6808/3430112748,v1:172.21.15.172:6809/3430112748]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T15:41:15.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:15 smithi145 ceph-mon[88162]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T15:41:15.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:15 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:15.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:15 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:15.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:15 smithi172 ceph-mon[65307]: pgmap v216: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 71/627 objects degraded (11.324%) 2024-04-22T15:41:15.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:15 smithi172 ceph-mon[65307]: from='osd.5 [v2:172.21.15.172:6808/3430112748,v1:172.21.15.172:6809/3430112748]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T15:41:15.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:15 smithi172 ceph-mon[65307]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T15:41:15.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:15 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:15.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:15 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:16.217 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:41:15 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:41:15.876+0000 7fc583c9c700 -1 osd.5 133 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T15:41:16.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:16 smithi145 ceph-mon[82523]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-22T15:41:16.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:16 smithi145 ceph-mon[82523]: from='osd.5 [v2:172.21.15.172:6808/3430112748,v1:172.21.15.172:6809/3430112748]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:41:16.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:16 smithi145 ceph-mon[82523]: osdmap e136: 8 total, 7 up, 8 in 2024-04-22T15:41:16.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:16 smithi145 ceph-mon[82523]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:41:16.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:16 smithi145 ceph-mon[82523]: from='osd.5 ' entity='osd.5' 2024-04-22T15:41:16.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:16 smithi145 ceph-mon[88162]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-22T15:41:16.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:16 smithi145 ceph-mon[88162]: from='osd.5 [v2:172.21.15.172:6808/3430112748,v1:172.21.15.172:6809/3430112748]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:41:16.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:16 smithi145 ceph-mon[88162]: osdmap e136: 8 total, 7 up, 8 in 2024-04-22T15:41:16.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:16 smithi145 ceph-mon[88162]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:41:16.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:16 smithi145 ceph-mon[88162]: from='osd.5 ' entity='osd.5' 2024-04-22T15:41:16.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:16 smithi172 ceph-mon[65307]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-22T15:41:16.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:16 smithi172 ceph-mon[65307]: from='osd.5 [v2:172.21.15.172:6808/3430112748,v1:172.21.15.172:6809/3430112748]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:41:16.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:16 smithi172 ceph-mon[65307]: osdmap e136: 8 total, 7 up, 8 in 2024-04-22T15:41:16.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:16 smithi172 ceph-mon[65307]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:41:16.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:16 smithi172 ceph-mon[65307]: from='osd.5 ' entity='osd.5' 2024-04-22T15:41:17.600 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:17 smithi172 ceph-mon[65307]: pgmap v218: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 201 MiB used, 715 GiB / 715 GiB avail; 71/627 objects degraded (11.324%) 2024-04-22T15:41:17.600 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:17 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:17.600 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:17 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:17.600 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:17 smithi172 ceph-mon[65307]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:41:17.600 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:17 smithi172 ceph-mon[65307]: osd.5 [v2:172.21.15.172:6808/3430112748,v1:172.21.15.172:6809/3430112748] boot 2024-04-22T15:41:17.600 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:17 smithi172 ceph-mon[65307]: osdmap e137: 8 total, 8 up, 8 in 2024-04-22T15:41:17.600 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:17 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:41:17.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:17 smithi145 ceph-mon[88162]: pgmap v218: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 201 MiB used, 715 GiB / 715 GiB avail; 71/627 objects degraded (11.324%) 2024-04-22T15:41:17.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:17 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:17.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:17 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:17.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:17 smithi145 ceph-mon[88162]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:41:17.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:17 smithi145 ceph-mon[88162]: osd.5 [v2:172.21.15.172:6808/3430112748,v1:172.21.15.172:6809/3430112748] boot 2024-04-22T15:41:17.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:17 smithi145 ceph-mon[88162]: osdmap e137: 8 total, 8 up, 8 in 2024-04-22T15:41:17.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:17 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:41:17.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:17 smithi145 ceph-mon[82523]: pgmap v218: 161 pgs: 20 active+undersized+degraded, 39 active+undersized, 102 active+clean; 585 KiB data, 201 MiB used, 715 GiB / 715 GiB avail; 71/627 objects degraded (11.324%) 2024-04-22T15:41:17.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:17 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:17.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:17 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:17.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:17 smithi145 ceph-mon[82523]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:41:17.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:17 smithi145 ceph-mon[82523]: osd.5 [v2:172.21.15.172:6808/3430112748,v1:172.21.15.172:6809/3430112748] boot 2024-04-22T15:41:17.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:17 smithi145 ceph-mon[82523]: osdmap e137: 8 total, 8 up, 8 in 2024-04-22T15:41:17.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:17 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T15:41:19.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:18 smithi172 ceph-mon[65307]: osdmap e138: 8 total, 8 up, 8 in 2024-04-22T15:41:19.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:18 smithi172 ceph-mon[65307]: pgmap v221: 161 pgs: 5 peering, 14 active+undersized+degraded, 27 active+undersized, 115 active+clean; 585 KiB data, 201 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s; 46/627 objects degraded (7.337%) 2024-04-22T15:41:19.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:18 smithi145 ceph-mon[82523]: osdmap e138: 8 total, 8 up, 8 in 2024-04-22T15:41:19.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:18 smithi145 ceph-mon[82523]: pgmap v221: 161 pgs: 5 peering, 14 active+undersized+degraded, 27 active+undersized, 115 active+clean; 585 KiB data, 201 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s; 46/627 objects degraded (7.337%) 2024-04-22T15:41:19.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:18 smithi145 ceph-mon[88162]: osdmap e138: 8 total, 8 up, 8 in 2024-04-22T15:41:19.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:18 smithi145 ceph-mon[88162]: pgmap v221: 161 pgs: 5 peering, 14 active+undersized+degraded, 27 active+undersized, 115 active+clean; 585 KiB data, 201 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s; 46/627 objects degraded (7.337%) 2024-04-22T15:41:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:20 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:20 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:41:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:20 smithi145 ceph-mon[82523]: Health check update: Degraded data redundancy: 46/627 objects degraded (7.337%), 14 pgs degraded (PG_DEGRADED) 2024-04-22T15:41:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:20 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:20 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:20 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:41:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:20 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:41:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:20 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:20 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:41:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:20 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:41:20.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:20 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:20.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:20 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:41:20.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:20 smithi145 ceph-mon[88162]: Health check update: Degraded data redundancy: 46/627 objects degraded (7.337%), 14 pgs degraded (PG_DEGRADED) 2024-04-22T15:41:20.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:20 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:20.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:20 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:20.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:20 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:41:20.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:20 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:41:20.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:20 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:20.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:20 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:41:20.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:20 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:41:20.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:20 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:20.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:20 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:41:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:20 smithi172 ceph-mon[65307]: Health check update: Degraded data redundancy: 46/627 objects degraded (7.337%), 14 pgs degraded (PG_DEGRADED) 2024-04-22T15:41:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:20 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:20 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:20 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:41:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:20 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:41:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:20 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:20 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:41:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:20 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:41:21.576 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-mon[82523]: pgmap v222: 161 pgs: 5 peering, 14 active+undersized+degraded, 23 active+undersized, 119 active+clean; 585 KiB data, 201 MiB used, 715 GiB / 715 GiB avail; 751 B/s rd, 0 op/s; 46/627 objects degraded (7.337%) 2024-04-22T15:41:21.576 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-mon[82523]: pgmap v223: 161 pgs: 5 peering, 14 active+undersized+degraded, 23 active+undersized, 119 active+clean; 585 KiB data, 201 MiB used, 715 GiB / 715 GiB avail; 929 B/s rd, 0 op/s; 46/627 objects degraded (7.337%) 2024-04-22T15:41:21.576 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-mon[82523]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:41:21.576 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-mon[82523]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:41:21.577 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-mon[88162]: pgmap v222: 161 pgs: 5 peering, 14 active+undersized+degraded, 23 active+undersized, 119 active+clean; 585 KiB data, 201 MiB used, 715 GiB / 715 GiB avail; 751 B/s rd, 0 op/s; 46/627 objects degraded (7.337%) 2024-04-22T15:41:21.577 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-mon[88162]: pgmap v223: 161 pgs: 5 peering, 14 active+undersized+degraded, 23 active+undersized, 119 active+clean; 585 KiB data, 201 MiB used, 715 GiB / 715 GiB avail; 929 B/s rd, 0 op/s; 46/627 objects degraded (7.337%) 2024-04-22T15:41:21.577 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-mon[88162]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:41:21.577 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-mon[88162]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:41:21.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:21 smithi172 ceph-mon[65307]: pgmap v222: 161 pgs: 5 peering, 14 active+undersized+degraded, 23 active+undersized, 119 active+clean; 585 KiB data, 201 MiB used, 715 GiB / 715 GiB avail; 751 B/s rd, 0 op/s; 46/627 objects degraded (7.337%) 2024-04-22T15:41:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:21 smithi172 ceph-mon[65307]: pgmap v223: 161 pgs: 5 peering, 14 active+undersized+degraded, 23 active+undersized, 119 active+clean; 585 KiB data, 201 MiB used, 715 GiB / 715 GiB avail; 929 B/s rd, 0 op/s; 46/627 objects degraded (7.337%) 2024-04-22T15:41:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:21 smithi172 ceph-mon[65307]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:41:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:21 smithi172 ceph-mon[65307]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:41:21.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:41:21.731+0000 7f17b18ae700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:41:21.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:41:21.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:41:21.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:41:21.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:41:21.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:41:21.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:41:21.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:41:21.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:41:21.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:41:21.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:41:21.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:41:21.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:41:21.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:41:21.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:41:21.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:41:21.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:41:21.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:41:21.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:41:21.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:41:21.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:41:21.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:41:21.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:41:21.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:41:21.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:41:21.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:41:21.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:41:21.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:41:21.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:41:21.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:41:21.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:41:21.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:41:21.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:41:21.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:41:21.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:41:21.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:41:21.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:41:21.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:41:21.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:41:21.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:41:21.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:41:21.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:41:21.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:41:21.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:41:21.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:41:21.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:41:21.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:41:21.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:41:21.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:41:21.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:41:21.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:41:21.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:41:21.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:41:21.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:41:21.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:41:21.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:41:21.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:41:21.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:41:21.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:41:21.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:21 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:41:21.742+0000 7f17e84c2700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (17 PGs are or would become offline) 2024-04-22T15:41:22.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:22 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:41:22.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:22 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:41:22.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:22 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:41:22.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:22 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:41:22.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:22 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:41:22.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:22 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T15:41:22.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:22 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:41:22.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:22 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:41:22.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:22 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:41:22.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:22 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:41:22.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:22 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:41:22.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:22 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T15:41:22.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:22 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:41:22.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:22 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:41:22.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:22 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:41:22.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:22 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:41:22.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:22 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:41:22.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:22 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T15:41:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:23 smithi145 ceph-mon[82523]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:41:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:41:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:41:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:41:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:41:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:41:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:41:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:41:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:41:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:41:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:41:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:41:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:41:23.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:41:23.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:41:23.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:41:23.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:41:23.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:41:23.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:41:23.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:41:23.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:41:23.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:41:23.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:41:23.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:41:23.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:41:23.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:41:23.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:41:23.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:41:23.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:41:23.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:41:23.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:41:23.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:41:23.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:41:23.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:41:23.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:41:23.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:41:23.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:41:23.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:41:23.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:41:23.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:41:23.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:41:23.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:41:23.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:41:23.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:41:23.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:41:23.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:41:23.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:41:23.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:41:23.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:41:23.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:41:23.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:41:23.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:41:23.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:41:23.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:41:23.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:23 smithi145 ceph-mon[88162]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:41:23.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:41:23.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:41:23.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:41:23.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:41:23.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:41:23.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:41:23.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:41:23.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:41:23.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:41:23.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:41:23.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:41:23.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:41:23.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:41:23.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:41:23.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:41:23.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:41:23.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:41:23.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:41:23.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:41:23.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:41:23.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:41:23.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:41:23.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:41:23.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:41:23.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:41:23.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:41:23.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:41:23.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:41:23.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:41:23.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:41:23.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:41:23.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:41:23.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:41:23.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:41:23.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:41:23.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:41:23.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:41:23.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:41:23.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:41:23.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:41:23.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:41:23.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:41:23.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:41:23.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:41:23.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:41:23.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:41:23.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:41:23.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:41:23.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:41:23.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:41:23.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:41:23.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:41:23.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:41:23.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:41:23.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:41:23.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:41:23.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:41:23.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:41:23.631 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:23 smithi145 ceph-mon[82523]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T15:41:23.631 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:23 smithi145 ceph-mon[82523]: Upgrade: unsafe to stop osd(s) at this time (17 PGs are or would become offline) 2024-04-22T15:41:23.631 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:23 smithi145 ceph-mon[82523]: pgmap v224: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 751 B/s rd, 0 op/s 2024-04-22T15:41:23.631 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:23 smithi145 ceph-mon[82523]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 46/627 objects degraded (7.337%), 14 pgs degraded) 2024-04-22T15:41:23.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:41:23.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:41:23.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:41:23.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:41:23.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:41:23.631 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:23 smithi145 ceph-mon[88162]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T15:41:23.631 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:23 smithi145 ceph-mon[88162]: Upgrade: unsafe to stop osd(s) at this time (17 PGs are or would become offline) 2024-04-22T15:41:23.632 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:23 smithi145 ceph-mon[88162]: pgmap v224: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 751 B/s rd, 0 op/s 2024-04-22T15:41:23.632 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:23 smithi145 ceph-mon[88162]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 46/627 objects degraded (7.337%), 14 pgs degraded) 2024-04-22T15:41:23.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:23 smithi172 ceph-mon[65307]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:41:23.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:41:23.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:41:23.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:41:23.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:41:23.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:41:23.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:41:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:41:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:41:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:41:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:41:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:41:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:41:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:41:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:41:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:41:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:41:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:41:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:41:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:41:23.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:41:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:41:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:41:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:41:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:41:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:41:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:41:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:41:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:41:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:41:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:41:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:41:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:41:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:41:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:41:23.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:41:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:41:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:41:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:41:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:41:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:41:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:41:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:41:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:41:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:41:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:41:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:41:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:41:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:41:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:41:23.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:41:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:41:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:41:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:41:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:41:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:41:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:41:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:41:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:41:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:23 smithi172 ceph-mon[65307]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T15:41:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:23 smithi172 ceph-mon[65307]: Upgrade: unsafe to stop osd(s) at this time (17 PGs are or would become offline) 2024-04-22T15:41:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:23 smithi172 ceph-mon[65307]: pgmap v224: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 751 B/s rd, 0 op/s 2024-04-22T15:41:23.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:23 smithi172 ceph-mon[65307]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 46/627 objects degraded (7.337%), 14 pgs degraded) 2024-04-22T15:41:25.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:25 smithi145 ceph-mon[82523]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:41:25.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:25 smithi145 ceph-mon[88162]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:41:25.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:41:25] "GET /metrics HTTP/1.1" 200 38220 "" "Prometheus/2.43.0" 2024-04-22T15:41:25.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:25 smithi172 ceph-mon[65307]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:41:27.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:27 smithi145 ceph-mon[88162]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:41:27.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:27 smithi145 ceph-mon[82523]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:41:27.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:27 smithi172 ceph-mon[65307]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:41:29.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:29 smithi145 ceph-mon[88162]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 541 B/s rd, 0 op/s 2024-04-22T15:41:29.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:29 smithi145 ceph-mon[82523]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 541 B/s rd, 0 op/s 2024-04-22T15:41:29.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:29 smithi172 ceph-mon[65307]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 541 B/s rd, 0 op/s 2024-04-22T15:41:30.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:30 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:30.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:30 smithi145 ceph-mon[82523]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:41:30.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:30 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:30.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:30 smithi145 ceph-mon[88162]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:41:30.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:30 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:30.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:30 smithi172 ceph-mon[65307]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:41:31.921 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:31 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:41:31.922 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:31 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:41:31.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:31 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:41:32.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:32 smithi172 ceph-mon[65307]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:41:33.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:32 smithi145 ceph-mon[82523]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:41:33.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:32 smithi145 ceph-mon[88162]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:41:35.472 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:41:35.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:35 smithi145 ceph-mon[88162]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:41:35.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:35.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:41:35.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:41:35] "GET /metrics HTTP/1.1" 200 38216 "" "Prometheus/2.43.0" 2024-04-22T15:41:35.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:35 smithi145 ceph-mon[82523]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:41:35.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:35.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:41:35.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:35 smithi172 ceph-mon[65307]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:41:35.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:35.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:41:36.125 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:41:36.125 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (8m) 77s ago 30m 27.0M - 0.25.0 c8568f914cd2 1c58ccf37ffd 2024-04-22T15:41:36.125 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (8m) 20s ago 30m 77.5M - 9.4.12 2bacad6d85d8 2f825d15ab10 2024-04-22T15:41:36.125 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 unknown 77s ago 29m 60.7M - 3.5 e1d6a67b021e 9704631ffa0e 2024-04-22T15:41:36.125 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443,9283,8765 running (9m) 20s ago 35m 453M - 19.0.0-3223-g5883d494 3d983c26e278 e7500ef71604 2024-04-22T15:41:36.125 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (19m) 77s ago 36m 511M - 19.0.0-3223-g5883d494 3d983c26e278 b2de97566cf4 2024-04-22T15:41:36.125 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (5m) 77s ago 37m 53.7M 2048M 19.0.0-3223-g5883d494 3d983c26e278 7bfdc11120f4 2024-04-22T15:41:36.125 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (6m) 20s ago 35m 51.0M 2048M 19.0.0-3223-g5883d494 3d983c26e278 0a360b08a6d6 2024-04-22T15:41:36.125 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (5m) 77s ago 35m 42.0M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38dcfa677c18 2024-04-22T15:41:36.126 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (9m) 77s ago 31m 15.3M - 1.5.0 0da6a335fe13 8c75c26fec05 2024-04-22T15:41:36.126 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (9m) 20s ago 31m 16.3M - 1.5.0 0da6a335fe13 8fd2949e2eaf 2024-04-22T15:41:36.126 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (3m) 77s ago 35m 51.7M 2992M 19.0.0-3223-g5883d494 3d983c26e278 e4bc06317af0 2024-04-22T15:41:36.126 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (3m) 77s ago 34m 44.0M 2992M 19.0.0-3223-g5883d494 3d983c26e278 d299f622f08d 2024-04-22T15:41:36.126 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (2m) 77s ago 34m 41.7M 2992M 19.0.0-3223-g5883d494 3d983c26e278 6aea4a926fe4 2024-04-22T15:41:36.127 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (82s) 77s ago 33m 15.1M 2992M 19.0.0-3223-g5883d494 3d983c26e278 c0cd2bdc1439 2024-04-22T15:41:36.127 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (60s) 20s ago 33m 44.3M 3248M 19.0.0-3223-g5883d494 3d983c26e278 7ac5776bc07a 2024-04-22T15:41:36.127 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (25s) 20s ago 33m 12.2M 3248M 19.0.0-3223-g5883d494 3d983c26e278 5f803668a17b 2024-04-22T15:41:36.127 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (32m) 20s ago 32m 59.0M 3248M 17.2.0 e1d6a67b021e 45d475edf1ca 2024-04-22T15:41:36.127 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (32m) 20s ago 32m 59.4M 3248M 17.2.0 e1d6a67b021e cc9e1d9e7916 2024-04-22T15:41:36.127 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (8m) 20s ago 31m 70.5M - 2.43.0 a07b618ecd1d 51cf4df6e8d4 2024-04-22T15:41:36.127 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (29m) 77s ago 29m 80.2M - 17.2.0 e1d6a67b021e c90a636c947e 2024-04-22T15:41:36.128 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (29m) 20s ago 29m 78.1M - 17.2.0 e1d6a67b021e 20ac46e9bfda 2024-04-22T15:41:36.525 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:41:36.525 INFO:teuthology.orchestra.run.smithi145.stdout: "mon": { 2024-04-22T15:41:36.525 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 3 2024-04-22T15:41:36.526 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:41:36.526 INFO:teuthology.orchestra.run.smithi145.stdout: "mgr": { 2024-04-22T15:41:36.526 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:41:36.526 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:41:36.526 INFO:teuthology.orchestra.run.smithi145.stdout: "osd": { 2024-04-22T15:41:36.526 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-04-22T15:41:36.526 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 6 2024-04-22T15:41:36.526 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:41:36.526 INFO:teuthology.orchestra.run.smithi145.stdout: "rgw": { 2024-04-22T15:41:36.526 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:41:36.526 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:41:36.526 INFO:teuthology.orchestra.run.smithi145.stdout: "overall": { 2024-04-22T15:41:36.526 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-04-22T15:41:36.527 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 11 2024-04-22T15:41:36.527 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:41:36.527 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:41:36.861 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:41:36.862 INFO:teuthology.orchestra.run.smithi145.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:41:36.862 INFO:teuthology.orchestra.run.smithi145.stdout: "in_progress": true, 2024-04-22T15:41:36.862 INFO:teuthology.orchestra.run.smithi145.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-04-22T15:41:36.862 INFO:teuthology.orchestra.run.smithi145.stdout: "services_complete": [], 2024-04-22T15:41:36.862 INFO:teuthology.orchestra.run.smithi145.stdout: "progress": "6/8 daemons upgraded", 2024-04-22T15:41:36.862 INFO:teuthology.orchestra.run.smithi145.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T15:41:36.862 INFO:teuthology.orchestra.run.smithi145.stdout: "is_paused": false 2024-04-22T15:41:36.862 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:41:37.610 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:37 smithi172 ceph-mon[65307]: from='client.44541 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:41:37.610 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:37 smithi172 ceph-mon[65307]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:41:37.610 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:37 smithi172 ceph-mon[65307]: from='client.44547 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:41:37.610 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:37 smithi172 ceph-mon[65307]: from='client.44553 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:41:37.610 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:37 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/1079821695' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:41:37.610 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:37 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T15:41:37.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:37 smithi145 ceph-mon[88162]: from='client.44541 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:41:37.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:37 smithi145 ceph-mon[88162]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:41:37.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:37 smithi145 ceph-mon[88162]: from='client.44547 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:41:37.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:37 smithi145 ceph-mon[88162]: from='client.44553 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:41:37.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:37 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/1079821695' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:41:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:37 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T15:41:37.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:37 smithi145 ceph-mon[82523]: from='client.44541 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:41:37.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:37 smithi145 ceph-mon[82523]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:41:37.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:37 smithi145 ceph-mon[82523]: from='client.44547 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:41:37.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:37 smithi145 ceph-mon[82523]: from='client.44553 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:41:37.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:37 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/1079821695' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:41:37.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:37 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T15:41:38.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:38 smithi145 ceph-mon[82523]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T15:41:38.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:38 smithi145 ceph-mon[82523]: Upgrade: osd.6 is safe to restart 2024-04-22T15:41:38.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:38 smithi145 ceph-mon[82523]: from='client.44565 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:41:38.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:38 smithi145 ceph-mon[88162]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T15:41:38.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:38 smithi145 ceph-mon[88162]: Upgrade: osd.6 is safe to restart 2024-04-22T15:41:38.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:38 smithi145 ceph-mon[88162]: from='client.44565 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:41:38.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:38 smithi172 ceph-mon[65307]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-22T15:41:38.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:38 smithi172 ceph-mon[65307]: Upgrade: osd.6 is safe to restart 2024-04-22T15:41:38.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:38 smithi172 ceph-mon[65307]: from='client.44565 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:41:39.631 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:39 smithi172 ceph-mon[65307]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 0 B/s wr, 2 op/s 2024-04-22T15:41:39.631 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:39 smithi172 ceph-mon[65307]: Upgrade: Updating osd.6 2024-04-22T15:41:39.631 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:39 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:39.631 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:39 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T15:41:39.631 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:39 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:41:39.631 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:39 smithi172 ceph-mon[65307]: Deploying daemon osd.6 on smithi172 2024-04-22T15:41:39.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:39 smithi145 ceph-mon[82523]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 0 B/s wr, 2 op/s 2024-04-22T15:41:39.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:39 smithi145 ceph-mon[82523]: Upgrade: Updating osd.6 2024-04-22T15:41:39.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:39 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:39.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:39 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T15:41:39.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:39 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:41:39.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:39 smithi145 ceph-mon[82523]: Deploying daemon osd.6 on smithi172 2024-04-22T15:41:39.875 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:39 smithi145 ceph-mon[88162]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 0 B/s wr, 2 op/s 2024-04-22T15:41:39.875 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:39 smithi145 ceph-mon[88162]: Upgrade: Updating osd.6 2024-04-22T15:41:39.875 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:39 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:39.875 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:39 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T15:41:39.875 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:39 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:41:39.875 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:39 smithi145 ceph-mon[88162]: Deploying daemon osd.6 on smithi172 2024-04-22T15:41:41.660 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:41 smithi172 ceph-mon[65307]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 0 B/s wr, 5 op/s 2024-04-22T15:41:41.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:41 smithi145 ceph-mon[88162]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 0 B/s wr, 5 op/s 2024-04-22T15:41:41.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:41 smithi145 ceph-mon[82523]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 0 B/s wr, 5 op/s 2024-04-22T15:41:41.925 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:41 smithi172 systemd[1]: Stopping Ceph osd.6 for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:41:42.217 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:41 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[31955]: 2024-04-22T15:41:41.925+0000 7fe78cab0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:41:42.218 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:41 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[31955]: 2024-04-22T15:41:41.925+0000 7fe78cab0700 -1 osd.6 138 *** Got signal Terminated *** 2024-04-22T15:41:42.218 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:41 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[31955]: 2024-04-22T15:41:41.925+0000 7fe78cab0700 -1 osd.6 138 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T15:41:42.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:42 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:41:42.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:42 smithi172 ceph-mon[65307]: osd.6 marked itself down and dead 2024-04-22T15:41:42.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:42 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:41:42.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:42 smithi145 ceph-mon[82523]: osd.6 marked itself down and dead 2024-04-22T15:41:42.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:42 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:41:42.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:42 smithi145 ceph-mon[88162]: osd.6 marked itself down and dead 2024-04-22T15:41:43.112 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:42 smithi172 podman[85471]: 2024-04-22 15:41:42.751569303 +0000 UTC m=+0.925466028 container died 45d475edf1ca2d7541955db3a9bb21f4f0a4493e1f4010d365dda98aac4f103f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, GIT_CLEAN=True, architecture=x86_64, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, version=8, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream) 2024-04-22T15:41:43.401 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:43 smithi172 podman[85471]: 2024-04-22 15:41:43.109425115 +0000 UTC m=+1.283321840 container cleanup 45d475edf1ca2d7541955db3a9bb21f4f0a4493e1f4010d365dda98aac4f103f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, io.buildah.version=1.19.8, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, release=754, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, name=centos-stream, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, version=8, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git) 2024-04-22T15:41:43.401 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:43 smithi172 bash[85471]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6 2024-04-22T15:41:43.401 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:43 smithi172 podman[85497]: 2024-04-22 15:41:43.277519964 +0000 UTC m=+0.524122341 container remove 45d475edf1ca2d7541955db3a9bb21f4f0a4493e1f4010d365dda98aac4f103f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, io.openshift.expose-services=, release=754, vcs-type=git, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, architecture=x86_64, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, distribution-scope=public, version=8, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , ceph=True, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-22T15:41:43.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:43 smithi172 ceph-mon[65307]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 37 op/s 2024-04-22T15:41:43.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:43 smithi172 ceph-mon[65307]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:41:43.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:43 smithi172 ceph-mon[65307]: osdmap e139: 8 total, 7 up, 8 in 2024-04-22T15:41:43.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:43 smithi145 ceph-mon[88162]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 37 op/s 2024-04-22T15:41:43.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:43 smithi145 ceph-mon[88162]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:41:43.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:43 smithi145 ceph-mon[88162]: osdmap e139: 8 total, 7 up, 8 in 2024-04-22T15:41:43.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:43 smithi145 ceph-mon[82523]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 37 op/s 2024-04-22T15:41:43.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:43 smithi145 ceph-mon[82523]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:41:43.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:43 smithi145 ceph-mon[82523]: osdmap e139: 8 total, 7 up, 8 in 2024-04-22T15:41:44.049 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:43 smithi172 podman[85593]: 2024-04-22 15:41:43.728521379 +0000 UTC m=+0.107739647 container create 18a850a00c0bbb20c1593684746ef13ed62dad7a2be113f9fb03a63ab386c3ae (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416) 2024-04-22T15:41:44.049 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:43 smithi172 podman[85593]: 2024-04-22 15:41:43.639171031 +0000 UTC m=+0.018389303 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:41:44.050 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:43 smithi172 podman[85593]: 2024-04-22 15:41:43.964254467 +0000 UTC m=+0.343472738 container init 18a850a00c0bbb20c1593684746ef13ed62dad7a2be113f9fb03a63ab386c3ae (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-04-22T15:41:44.050 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:43 smithi172 podman[85593]: 2024-04-22 15:41:43.967863353 +0000 UTC m=+0.347081627 container start 18a850a00c0bbb20c1593684746ef13ed62dad7a2be113f9fb03a63ab386c3ae (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6-deactivate, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T15:41:44.050 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:44 smithi172 podman[85593]: 2024-04-22 15:41:44.0476903 +0000 UTC m=+0.426908584 container attach 18a850a00c0bbb20c1593684746ef13ed62dad7a2be113f9fb03a63ab386c3ae (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.build-date=20240416) 2024-04-22T15:41:44.403 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:44 smithi172 podman[85593]: 2024-04-22 15:41:44.267374319 +0000 UTC m=+0.646592589 container died 18a850a00c0bbb20c1593684746ef13ed62dad7a2be113f9fb03a63ab386c3ae (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6-deactivate, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416) 2024-04-22T15:41:44.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:44 smithi172 ceph-mon[65307]: osdmap e140: 8 total, 7 up, 8 in 2024-04-22T15:41:44.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:44 smithi172 ceph-mon[65307]: pgmap v237: 161 pgs: 18 peering, 9 stale+active+clean, 134 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 0 B/s wr, 132 op/s 2024-04-22T15:41:44.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:44 smithi145 ceph-mon[88162]: osdmap e140: 8 total, 7 up, 8 in 2024-04-22T15:41:44.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:44 smithi145 ceph-mon[88162]: pgmap v237: 161 pgs: 18 peering, 9 stale+active+clean, 134 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 0 B/s wr, 132 op/s 2024-04-22T15:41:44.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:44 smithi145 ceph-mon[82523]: osdmap e140: 8 total, 7 up, 8 in 2024-04-22T15:41:44.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:44 smithi145 ceph-mon[82523]: pgmap v237: 161 pgs: 18 peering, 9 stale+active+clean, 134 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 0 B/s wr, 132 op/s 2024-04-22T15:41:45.218 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:44 smithi172 podman[85619]: 2024-04-22 15:41:44.860137441 +0000 UTC m=+0.588101035 container remove 18a850a00c0bbb20c1593684746ef13ed62dad7a2be113f9fb03a63ab386c3ae (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_CLEAN=True) 2024-04-22T15:41:45.218 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:44 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.6.service: Deactivated successfully. 2024-04-22T15:41:45.218 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:44 smithi172 systemd[1]: Stopped Ceph osd.6 for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:41:45.218 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:44 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.6.service: Consumed 15.180s CPU time. 2024-04-22T15:41:45.218 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:45 smithi172 systemd[1]: Starting Ceph osd.6 for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:41:45.607 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:45 smithi172 ceph-mon[65307]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-04-22T15:41:45.608 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:45 smithi172 podman[85723]: 2024-04-22 15:41:45.201972083 +0000 UTC m=+0.019026970 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:41:45.608 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:45 smithi172 podman[85723]: 2024-04-22 15:41:45.319954079 +0000 UTC m=+0.137009019 container create 42bb18141edc972cb8b8cdef2cc0a148bd914b7dd8ab4c6526974e493101ec4f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6-activate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True) 2024-04-22T15:41:45.608 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:45 smithi172 podman[85723]: 2024-04-22 15:41:45.530999857 +0000 UTC m=+0.348054750 container init 42bb18141edc972cb8b8cdef2cc0a148bd914b7dd8ab4c6526974e493101ec4f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6-activate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T15:41:45.608 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:45 smithi172 podman[85723]: 2024-04-22 15:41:45.534365863 +0000 UTC m=+0.351420750 container start 42bb18141edc972cb8b8cdef2cc0a148bd914b7dd8ab4c6526974e493101ec4f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6-activate, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-04-22T15:41:45.608 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:45 smithi172 podman[85723]: 2024-04-22 15:41:45.6054495 +0000 UTC m=+0.422504403 container attach 42bb18141edc972cb8b8cdef2cc0a148bd914b7dd8ab4c6526974e493101ec4f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6-activate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-04-22T15:41:45.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:45 smithi145 ceph-mon[88162]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-04-22T15:41:45.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:45 smithi145 ceph-mon[82523]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-04-22T15:41:45.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:41:45] "GET /metrics HTTP/1.1" 200 38216 "" "Prometheus/2.43.0" 2024-04-22T15:41:46.278 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:46 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6-activate[85738]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:41:46.278 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:46 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6-activate[85738]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:41:46.278 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:46 smithi172 bash[85723]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:41:46.278 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:46 smithi172 bash[85723]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:41:46.278 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:46 smithi172 bash[85723]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-22T15:41:46.278 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:46 smithi172 bash[85723]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-22T15:41:46.278 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:46 smithi172 bash[85723]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-22T15:41:46.278 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:46 smithi172 bash[85723]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-22T15:41:46.278 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:46 smithi172 bash[85723]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-04-22T15:41:46.278 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:46 smithi172 bash[85723]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-22T15:41:46.279 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:46 smithi172 bash[85723]: --> ceph-volume raw activate successful for osd ID: 6 2024-04-22T15:41:46.279 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:46 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6-activate[85738]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-22T15:41:46.279 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:46 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6-activate[85738]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-22T15:41:46.279 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:46 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6-activate[85738]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-22T15:41:46.279 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:46 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6-activate[85738]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-22T15:41:46.279 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:46 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6-activate[85738]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-04-22T15:41:46.279 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:46 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6-activate[85738]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-22T15:41:46.279 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:46 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6-activate[85738]: --> ceph-volume raw activate successful for osd ID: 6 2024-04-22T15:41:46.279 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:46 smithi172 podman[85723]: 2024-04-22 15:41:46.038232056 +0000 UTC m=+0.855286932 container died 42bb18141edc972cb8b8cdef2cc0a148bd914b7dd8ab4c6526974e493101ec4f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6-activate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-04-22T15:41:46.570 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:46 smithi172 ceph-mon[65307]: pgmap v238: 161 pgs: 11 active+undersized+degraded, 18 peering, 24 active+undersized, 108 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 106 KiB/s rd, 0 B/s wr, 176 op/s; 50/627 objects degraded (7.974%) 2024-04-22T15:41:46.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:46 smithi145 ceph-mon[88162]: pgmap v238: 161 pgs: 11 active+undersized+degraded, 18 peering, 24 active+undersized, 108 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 106 KiB/s rd, 0 B/s wr, 176 op/s; 50/627 objects degraded (7.974%) 2024-04-22T15:41:46.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:46 smithi145 ceph-mon[82523]: pgmap v238: 161 pgs: 11 active+undersized+degraded, 18 peering, 24 active+undersized, 108 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 106 KiB/s rd, 0 B/s wr, 176 op/s; 50/627 objects degraded (7.974%) 2024-04-22T15:41:46.916 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:46 smithi172 podman[85876]: 2024-04-22 15:41:46.568053274 +0000 UTC m=+0.522780033 container remove 42bb18141edc972cb8b8cdef2cc0a148bd914b7dd8ab4c6526974e493101ec4f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6-activate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-22T15:41:46.917 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:46 smithi172 podman[85942]: 2024-04-22 15:41:46.687915491 +0000 UTC m=+0.018883486 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:41:46.917 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:46 smithi172 podman[85942]: 2024-04-22 15:41:46.810769877 +0000 UTC m=+0.141737869 container create 9e049dd4678d36462e6608f5514e745938cf46a353247b51b627458aeb62b076 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.build-date=20240416) 2024-04-22T15:41:47.217 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:47 smithi172 podman[85942]: 2024-04-22 15:41:47.0216484 +0000 UTC m=+0.352616396 container init 9e049dd4678d36462e6608f5514e745938cf46a353247b51b627458aeb62b076 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T15:41:47.218 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:47 smithi172 podman[85942]: 2024-04-22 15:41:47.024695569 +0000 UTC m=+0.355663563 container start 9e049dd4678d36462e6608f5514e745938cf46a353247b51b627458aeb62b076 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T15:41:47.218 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:47 smithi172 ceph-osd[85961]: -- 172.21.15.172:0/4029390903 <== mon.2 v2:172.21.15.172:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5582ea811680 con 0x5582ea7dac00 2024-04-22T15:41:47.218 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:47 smithi172 bash[85942]: 9e049dd4678d36462e6608f5514e745938cf46a353247b51b627458aeb62b076 2024-04-22T15:41:47.218 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:47 smithi172 systemd[1]: Started Ceph osd.6 for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:41:47.717 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:41:47.570+0000 7f94638717c0 -1 Falling back to public interface 2024-04-22T15:41:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:47 smithi172 ceph-mon[65307]: Health check failed: Degraded data redundancy: 50/627 objects degraded (7.974%), 11 pgs degraded (PG_DEGRADED) 2024-04-22T15:41:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:47 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:47 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:47.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:47 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:41:47.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:47 smithi145 ceph-mon[82523]: Health check failed: Degraded data redundancy: 50/627 objects degraded (7.974%), 11 pgs degraded (PG_DEGRADED) 2024-04-22T15:41:47.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:47 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:47.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:47 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:47.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:47 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:41:47.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:47 smithi145 ceph-mon[88162]: Health check failed: Degraded data redundancy: 50/627 objects degraded (7.974%), 11 pgs degraded (PG_DEGRADED) 2024-04-22T15:41:47.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:47 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:47.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:47 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:47.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:47 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:41:48.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:48 smithi172 ceph-mon[65307]: pgmap v239: 161 pgs: 12 active+undersized+degraded, 15 peering, 26 active+undersized, 108 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 102 KiB/s rd, 0 B/s wr, 171 op/s; 52/627 objects degraded (8.293%) 2024-04-22T15:41:48.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:48 smithi145 ceph-mon[82523]: pgmap v239: 161 pgs: 12 active+undersized+degraded, 15 peering, 26 active+undersized, 108 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 102 KiB/s rd, 0 B/s wr, 171 op/s; 52/627 objects degraded (8.293%) 2024-04-22T15:41:48.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:48 smithi145 ceph-mon[88162]: pgmap v239: 161 pgs: 12 active+undersized+degraded, 15 peering, 26 active+undersized, 108 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 102 KiB/s rd, 0 B/s wr, 171 op/s; 52/627 objects degraded (8.293%) 2024-04-22T15:41:50.337 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:50 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:41:50.308+0000 7f94638717c0 -1 osd.6 0 read_superblock omap replica is missing. 2024-04-22T15:41:50.337 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:50 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:50.337 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:50 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:41:50.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:50 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:50.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:50 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:41:50.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:50 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:50.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:50 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:41:50.717 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:50 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:41:50.336+0000 7f94638717c0 -1 osd.6 138 log_to_monitors true 2024-04-22T15:41:51.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:51 smithi145 ceph-mon[82523]: pgmap v240: 161 pgs: 17 active+undersized+degraded, 36 active+undersized, 108 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s rd, 0 B/s wr, 123 op/s; 73/627 objects degraded (11.643%) 2024-04-22T15:41:51.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:51 smithi145 ceph-mon[82523]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-04-22T15:41:51.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:51 smithi145 ceph-mon[82523]: from='osd.6 [v2:172.21.15.172:6816/3052359651,v1:172.21.15.172:6817/3052359651]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T15:41:51.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:51 smithi145 ceph-mon[82523]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T15:41:51.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:51 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:51.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:51 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:51.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:51 smithi145 ceph-mon[88162]: pgmap v240: 161 pgs: 17 active+undersized+degraded, 36 active+undersized, 108 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s rd, 0 B/s wr, 123 op/s; 73/627 objects degraded (11.643%) 2024-04-22T15:41:51.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:51 smithi145 ceph-mon[88162]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-04-22T15:41:51.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:51 smithi145 ceph-mon[88162]: from='osd.6 [v2:172.21.15.172:6816/3052359651,v1:172.21.15.172:6817/3052359651]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T15:41:51.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:51 smithi145 ceph-mon[88162]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T15:41:51.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:51 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:51.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:51 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:51.717 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:41:51 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:41:51.339+0000 7f9454cc9700 -1 osd.6 138 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T15:41:51.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:51 smithi172 ceph-mon[65307]: pgmap v240: 161 pgs: 17 active+undersized+degraded, 36 active+undersized, 108 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s rd, 0 B/s wr, 123 op/s; 73/627 objects degraded (11.643%) 2024-04-22T15:41:51.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:51 smithi172 ceph-mon[65307]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-04-22T15:41:51.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:51 smithi172 ceph-mon[65307]: from='osd.6 [v2:172.21.15.172:6816/3052359651,v1:172.21.15.172:6817/3052359651]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T15:41:51.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:51 smithi172 ceph-mon[65307]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T15:41:51.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:51 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:51.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:51 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:52.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:52 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:41:52.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:52 smithi145 ceph-mon[88162]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-22T15:41:52.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:52 smithi145 ceph-mon[88162]: osdmap e141: 8 total, 7 up, 8 in 2024-04-22T15:41:52.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:52 smithi145 ceph-mon[88162]: from='osd.6 [v2:172.21.15.172:6816/3052359651,v1:172.21.15.172:6817/3052359651]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:41:52.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:52 smithi145 ceph-mon[88162]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:41:52.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:52 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:41:52.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:52 smithi145 ceph-mon[82523]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-22T15:41:52.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:52 smithi145 ceph-mon[82523]: osdmap e141: 8 total, 7 up, 8 in 2024-04-22T15:41:52.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:52 smithi145 ceph-mon[82523]: from='osd.6 [v2:172.21.15.172:6816/3052359651,v1:172.21.15.172:6817/3052359651]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:41:52.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:52 smithi145 ceph-mon[82523]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:41:52.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:52 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:41:52.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:52 smithi172 ceph-mon[65307]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-22T15:41:52.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:52 smithi172 ceph-mon[65307]: osdmap e141: 8 total, 7 up, 8 in 2024-04-22T15:41:52.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:52 smithi172 ceph-mon[65307]: from='osd.6 [v2:172.21.15.172:6816/3052359651,v1:172.21.15.172:6817/3052359651]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:41:52.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:52 smithi172 ceph-mon[65307]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:41:53.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:53 smithi172 ceph-mon[65307]: pgmap v242: 161 pgs: 17 active+undersized+degraded, 36 active+undersized, 108 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 44 op/s; 73/627 objects degraded (11.643%) 2024-04-22T15:41:53.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:53 smithi172 ceph-mon[65307]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:41:53.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:53 smithi172 ceph-mon[65307]: osd.6 [v2:172.21.15.172:6816/3052359651,v1:172.21.15.172:6817/3052359651] boot 2024-04-22T15:41:53.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:53 smithi172 ceph-mon[65307]: osdmap e142: 8 total, 8 up, 8 in 2024-04-22T15:41:53.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:53 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:41:53.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:53 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:53.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:53 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:53.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:53 smithi145 ceph-mon[82523]: pgmap v242: 161 pgs: 17 active+undersized+degraded, 36 active+undersized, 108 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 44 op/s; 73/627 objects degraded (11.643%) 2024-04-22T15:41:53.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:53 smithi145 ceph-mon[82523]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:41:53.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:53 smithi145 ceph-mon[82523]: osd.6 [v2:172.21.15.172:6816/3052359651,v1:172.21.15.172:6817/3052359651] boot 2024-04-22T15:41:53.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:53 smithi145 ceph-mon[82523]: osdmap e142: 8 total, 8 up, 8 in 2024-04-22T15:41:53.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:53 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:41:53.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:53 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:53.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:53 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:53.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:53 smithi145 ceph-mon[88162]: pgmap v242: 161 pgs: 17 active+undersized+degraded, 36 active+undersized, 108 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 44 op/s; 73/627 objects degraded (11.643%) 2024-04-22T15:41:53.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:53 smithi145 ceph-mon[88162]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:41:53.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:53 smithi145 ceph-mon[88162]: osd.6 [v2:172.21.15.172:6816/3052359651,v1:172.21.15.172:6817/3052359651] boot 2024-04-22T15:41:53.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:53 smithi145 ceph-mon[88162]: osdmap e142: 8 total, 8 up, 8 in 2024-04-22T15:41:53.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:53 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T15:41:53.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:53 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:53.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:53 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:54.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:54 smithi172 ceph-mon[65307]: osdmap e143: 8 total, 8 up, 8 in 2024-04-22T15:41:54.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:54 smithi172 ceph-mon[65307]: Health check update: Degraded data redundancy: 73/627 objects degraded (11.643%), 17 pgs degraded (PG_DEGRADED) 2024-04-22T15:41:54.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:54 smithi145 ceph-mon[82523]: osdmap e143: 8 total, 8 up, 8 in 2024-04-22T15:41:54.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:54 smithi145 ceph-mon[82523]: Health check update: Degraded data redundancy: 73/627 objects degraded (11.643%), 17 pgs degraded (PG_DEGRADED) 2024-04-22T15:41:54.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:54 smithi145 ceph-mon[88162]: osdmap e143: 8 total, 8 up, 8 in 2024-04-22T15:41:54.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:54 smithi145 ceph-mon[88162]: Health check update: Degraded data redundancy: 73/627 objects degraded (11.643%), 17 pgs degraded (PG_DEGRADED) 2024-04-22T15:41:55.430 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:55 smithi172 ceph-mon[65307]: pgmap v245: 161 pgs: 14 peering, 12 active+undersized+degraded, 27 active+undersized, 108 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s; 52/627 objects degraded (8.293%) 2024-04-22T15:41:55.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:55 smithi145 ceph-mon[88162]: pgmap v245: 161 pgs: 14 peering, 12 active+undersized+degraded, 27 active+undersized, 108 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s; 52/627 objects degraded (8.293%) 2024-04-22T15:41:55.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:55 smithi145 ceph-mon[82523]: pgmap v245: 161 pgs: 14 peering, 12 active+undersized+degraded, 27 active+undersized, 108 active+clean; 585 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s; 52/627 objects degraded (8.293%) 2024-04-22T15:41:55.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:41:55] "GET /metrics HTTP/1.1" 200 38287 "" "Prometheus/2.43.0" 2024-04-22T15:41:56.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:56 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:56.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:56 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:56.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:56 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:41:56.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:56 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:41:56.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:56 smithi145 ceph-mon[88162]: pgmap v246: 161 pgs: 14 peering, 4 active+undersized+degraded, 9 active+undersized, 134 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 539 B/s rd, 0 op/s; 13/627 objects degraded (2.073%) 2024-04-22T15:41:56.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:56 smithi145 ceph-mon[88162]: pgmap v247: 161 pgs: 14 peering, 4 active+undersized+degraded, 9 active+undersized, 134 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 737 B/s rd, 0 op/s; 13/627 objects degraded (2.073%) 2024-04-22T15:41:56.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:56 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:56.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:56 smithi145 ceph-mon[88162]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:41:56.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:56 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:41:56.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:56 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:41:56.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:56 smithi145 ceph-mon[88162]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:41:56.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:56 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:56.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:56 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:56.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:56 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:41:56.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:56 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:41:56.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:56 smithi145 ceph-mon[82523]: pgmap v246: 161 pgs: 14 peering, 4 active+undersized+degraded, 9 active+undersized, 134 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 539 B/s rd, 0 op/s; 13/627 objects degraded (2.073%) 2024-04-22T15:41:56.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:56 smithi145 ceph-mon[82523]: pgmap v247: 161 pgs: 14 peering, 4 active+undersized+degraded, 9 active+undersized, 134 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 737 B/s rd, 0 op/s; 13/627 objects degraded (2.073%) 2024-04-22T15:41:56.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:56 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:56.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:56 smithi145 ceph-mon[82523]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:41:56.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:56 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:41:56.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:56 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:41:56.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:56 smithi145 ceph-mon[82523]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:41:56.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:56 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:56.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:56 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:56.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:56 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:41:56.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:56 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:41:56.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:56 smithi172 ceph-mon[65307]: pgmap v246: 161 pgs: 14 peering, 4 active+undersized+degraded, 9 active+undersized, 134 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 539 B/s rd, 0 op/s; 13/627 objects degraded (2.073%) 2024-04-22T15:41:56.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:56 smithi172 ceph-mon[65307]: pgmap v247: 161 pgs: 14 peering, 4 active+undersized+degraded, 9 active+undersized, 134 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 737 B/s rd, 0 op/s; 13/627 objects degraded (2.073%) 2024-04-22T15:41:56.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:56 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:41:56.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:56 smithi172 ceph-mon[65307]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:41:56.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:56 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:41:56.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:56 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:41:56.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:56 smithi172 ceph-mon[65307]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:41:57.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:41:57.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:41:57.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:41:57.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:41:57.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T15:41:57.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:41:57.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:41:57.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:41:57.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:41:57.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T15:41:57.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:41:57.435+0000 7f17b18ae700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:41:57.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:41:57.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:41:57.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:41:57.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:41:57.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:41:57.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:41:57.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:41:57.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:41:57.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:41:57.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:41:57.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:41:57.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:41:57.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:41:57.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:41:57.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:41:57.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:41:57.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:41:57.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:41:57.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:41:57.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:41:57.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:41:57.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:41:57.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:41:57.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:41:57.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:41:57.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:41:57.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:41:57.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:41:57.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:41:57.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:41:57.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:41:57.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:41:57.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:41:57.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:41:57.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:41:57.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:41:57.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:41:57.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:41:57.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:41:57.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:41:57.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:41:57.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:41:57.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:41:57.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:41:57.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:41:57.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:41:57.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:41:57.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:41:57.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:41:57.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:41:57.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:41:57.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:41:57.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:41:57.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:41:57.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:41:57.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:41:57.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:41:57.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:41:57.628 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:41:57 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:41:57.446+0000 7f17e84c2700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-04-22T15:41:57.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:57 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:41:57.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:57 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:41:57.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:57 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:41:57.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:57 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:41:57.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:57 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T15:41:58.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:58 smithi145 ceph-mon[82523]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:41:58.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:41:58.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:41:58.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:41:58.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:41:58.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:41:58.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:41:58.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:41:58.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:41:58.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:41:58.873 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:41:58.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:41:58.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:41:58.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:41:58.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:41:58.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:41:58.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:41:58.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:41:58.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:41:58.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:41:58.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:41:58.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:41:58.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:41:58.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:41:58.874 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:41:58.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:41:58.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:41:58.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:41:58.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:41:58.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:41:58.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:41:58.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:41:58.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:41:58.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:41:58.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:41:58.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:41:58.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:41:58.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:41:58.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:41:58.875 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:41:58.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:41:58.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:41:58.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:41:58.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:41:58.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:41:58.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:41:58.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:41:58.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:41:58.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:41:58.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:41:58.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:41:58.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:41:58.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:41:58.876 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:41:58.877 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:58 smithi145 ceph-mon[88162]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:41:58.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:41:58.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:41:58.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:41:58.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:41:58.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:41:58.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:41:58.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:41:58.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:41:58.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:41:58.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:41:58.877 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:41:58.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:41:58.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:41:58.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:41:58.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:41:58.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:41:58.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:41:58.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:41:58.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:41:58.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:41:58.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:41:58.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:41:58.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:41:58.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:41:58.878 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:41:58.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:41:58.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:41:58.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:41:58.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:41:58.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:41:58.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:41:58.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:41:58.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:41:58.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:41:58.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:41:58.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:41:58.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:41:58.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:41:58.879 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:41:58.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:41:58.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:41:58.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:41:58.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:41:58.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:41:58.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:41:58.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:41:58.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:41:58.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:41:58.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:41:58.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:41:58.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:41:58.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:41:58.880 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:41:58.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:41:58.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:41:58.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:41:58.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:41:58.881 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:41:58.881 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:58 smithi145 ceph-mon[82523]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T15:41:58.881 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:58 smithi145 ceph-mon[82523]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-04-22T15:41:58.881 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:58 smithi145 ceph-mon[82523]: pgmap v248: 161 pgs: 13 peering, 148 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 539 B/s rd, 0 op/s 2024-04-22T15:41:58.881 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:41:58.881 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:41:58.882 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:41:58.882 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:41:58.882 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:41:58.882 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:58 smithi145 ceph-mon[88162]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T15:41:58.882 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:58 smithi145 ceph-mon[88162]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-04-22T15:41:58.882 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:58 smithi145 ceph-mon[88162]: pgmap v248: 161 pgs: 13 peering, 148 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 539 B/s rd, 0 op/s 2024-04-22T15:41:58.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:58 smithi172 ceph-mon[65307]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:41:58.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:41:58.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:41:58.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:41:58.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:41:58.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:41:58.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:41:58.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:41:58.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:41:58.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:41:58.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:41:58.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:41:58.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:41:58.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:41:58.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:41:58.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:41:58.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:41:58.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:41:58.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:41:58.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:41:58.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:41:58.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:41:58.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:41:58.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:41:58.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:41:58.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:41:58.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:41:58.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:41:58.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:41:58.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:41:58.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:41:58.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:41:58.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:41:58.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:41:58.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:41:58.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:41:58.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:41:58.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:41:58.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:41:58.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:41:58.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:41:58.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:41:58.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:41:58.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:41:58.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:41:58.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:41:58.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:41:58.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:41:58.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:41:58.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:41:58.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:41:58.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:41:58.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:41:58.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:41:58.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:41:58.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:41:58.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:41:58.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:41:58.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:41:58.972 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:58 smithi172 ceph-mon[65307]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T15:41:58.972 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:58 smithi172 ceph-mon[65307]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-04-22T15:41:58.972 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:58 smithi172 ceph-mon[65307]: pgmap v248: 161 pgs: 13 peering, 148 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 539 B/s rd, 0 op/s 2024-04-22T15:41:59.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:41:59 smithi145 ceph-mon[82523]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/627 objects degraded (2.073%), 4 pgs degraded) 2024-04-22T15:41:59.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:41:59 smithi145 ceph-mon[88162]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/627 objects degraded (2.073%), 4 pgs degraded) 2024-04-22T15:41:59.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:41:59 smithi172 ceph-mon[65307]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/627 objects degraded (2.073%), 4 pgs degraded) 2024-04-22T15:42:00.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:00 smithi145 ceph-mon[82523]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:42:00.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:00 smithi145 ceph-mon[88162]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:42:00.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:00 smithi172 ceph-mon[65307]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:42:01.794 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:01 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:42:01.795 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:01 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:42:01.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:01 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:42:02.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:02 smithi145 ceph-mon[82523]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 627 B/s rd, 0 op/s 2024-04-22T15:42:02.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:02 smithi145 ceph-mon[88162]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 627 B/s rd, 0 op/s 2024-04-22T15:42:02.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:02 smithi172 ceph-mon[65307]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 627 B/s rd, 0 op/s 2024-04-22T15:42:05.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:05 smithi145 ceph-mon[82523]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:42:05.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:05 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:05.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:05 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:42:05.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:05 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:05.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:05 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:42:05] "GET /metrics HTTP/1.1" 200 38302 "" "Prometheus/2.43.0" 2024-04-22T15:42:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:05 smithi145 ceph-mon[88162]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:42:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:05 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:05 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:42:05.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:05 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:05.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:05 smithi172 ceph-mon[65307]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:42:05.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:05 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:05.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:05 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:42:05.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:05 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:07.222 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:42:07.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:07 smithi145 ceph-mon[88162]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:42:07.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:07 smithi145 ceph-mon[82523]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:42:07.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:07 smithi172 ceph-mon[65307]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:42:07.881 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:42:07.881 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (8m) 108s ago 31m 27.0M - 0.25.0 c8568f914cd2 1c58ccf37ffd 2024-04-22T15:42:07.881 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (8m) 16s ago 30m 80.7M - 9.4.12 2bacad6d85d8 2f825d15ab10 2024-04-22T15:42:07.881 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 unknown 108s ago 30m 60.7M - 3.5 e1d6a67b021e 9704631ffa0e 2024-04-22T15:42:07.881 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443,9283,8765 running (10m) 16s ago 35m 453M - 19.0.0-3223-g5883d494 3d983c26e278 e7500ef71604 2024-04-22T15:42:07.882 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (20m) 108s ago 37m 511M - 19.0.0-3223-g5883d494 3d983c26e278 b2de97566cf4 2024-04-22T15:42:07.882 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (6m) 108s ago 37m 53.7M 2048M 19.0.0-3223-g5883d494 3d983c26e278 7bfdc11120f4 2024-04-22T15:42:07.882 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (7m) 16s ago 36m 56.6M 2048M 19.0.0-3223-g5883d494 3d983c26e278 0a360b08a6d6 2024-04-22T15:42:07.882 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (6m) 108s ago 36m 42.0M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38dcfa677c18 2024-04-22T15:42:07.882 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (9m) 108s ago 31m 15.3M - 1.5.0 0da6a335fe13 8c75c26fec05 2024-04-22T15:42:07.882 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (9m) 16s ago 31m 16.1M - 1.5.0 0da6a335fe13 8fd2949e2eaf 2024-04-22T15:42:07.882 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (4m) 108s ago 35m 51.7M 2992M 19.0.0-3223-g5883d494 3d983c26e278 e4bc06317af0 2024-04-22T15:42:07.882 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (3m) 108s ago 35m 44.0M 2992M 19.0.0-3223-g5883d494 3d983c26e278 d299f622f08d 2024-04-22T15:42:07.882 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (2m) 108s ago 34m 41.7M 2992M 19.0.0-3223-g5883d494 3d983c26e278 6aea4a926fe4 2024-04-22T15:42:07.882 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (113s) 108s ago 34m 15.1M 2992M 19.0.0-3223-g5883d494 3d983c26e278 c0cd2bdc1439 2024-04-22T15:42:07.882 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (92s) 16s ago 33m 46.6M 3248M 19.0.0-3223-g5883d494 3d983c26e278 7ac5776bc07a 2024-04-22T15:42:07.882 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (57s) 16s ago 33m 52.7M 3248M 19.0.0-3223-g5883d494 3d983c26e278 5f803668a17b 2024-04-22T15:42:07.883 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (21s) 16s ago 33m 15.1M 3248M 19.0.0-3223-g5883d494 3d983c26e278 9e049dd4678d 2024-04-22T15:42:07.883 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (32m) 16s ago 32m 60.3M 3248M 17.2.0 e1d6a67b021e cc9e1d9e7916 2024-04-22T15:42:07.883 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (9m) 16s ago 32m 76.9M - 2.43.0 a07b618ecd1d 51cf4df6e8d4 2024-04-22T15:42:07.883 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (30m) 108s ago 30m 80.2M - 17.2.0 e1d6a67b021e c90a636c947e 2024-04-22T15:42:07.883 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (30m) 16s ago 30m 78.5M - 17.2.0 e1d6a67b021e 20ac46e9bfda 2024-04-22T15:42:08.276 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:42:08.276 INFO:teuthology.orchestra.run.smithi145.stdout: "mon": { 2024-04-22T15:42:08.277 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 3 2024-04-22T15:42:08.277 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:42:08.277 INFO:teuthology.orchestra.run.smithi145.stdout: "mgr": { 2024-04-22T15:42:08.277 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:42:08.277 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:42:08.277 INFO:teuthology.orchestra.run.smithi145.stdout: "osd": { 2024-04-22T15:42:08.277 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-22T15:42:08.277 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 7 2024-04-22T15:42:08.277 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:42:08.277 INFO:teuthology.orchestra.run.smithi145.stdout: "rgw": { 2024-04-22T15:42:08.277 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:42:08.277 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:42:08.277 INFO:teuthology.orchestra.run.smithi145.stdout: "overall": { 2024-04-22T15:42:08.277 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-04-22T15:42:08.277 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 12 2024-04-22T15:42:08.278 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:42:08.278 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:42:08.611 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:42:08.612 INFO:teuthology.orchestra.run.smithi145.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:42:08.612 INFO:teuthology.orchestra.run.smithi145.stdout: "in_progress": true, 2024-04-22T15:42:08.612 INFO:teuthology.orchestra.run.smithi145.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-04-22T15:42:08.612 INFO:teuthology.orchestra.run.smithi145.stdout: "services_complete": [], 2024-04-22T15:42:08.612 INFO:teuthology.orchestra.run.smithi145.stdout: "progress": "7/8 daemons upgraded", 2024-04-22T15:42:08.612 INFO:teuthology.orchestra.run.smithi145.stdout: "message": "Currently upgrading osd daemons", 2024-04-22T15:42:08.612 INFO:teuthology.orchestra.run.smithi145.stdout: "is_paused": false 2024-04-22T15:42:08.612 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:42:08.612 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:08 smithi145 ceph-mon[88162]: from='client.44577 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:42:08.613 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:08 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/1434043189' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:08.613 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:08 smithi145 ceph-mon[82523]: from='client.44577 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:42:08.613 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:08 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/1434043189' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:08.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:08 smithi172 ceph-mon[65307]: from='client.44577 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:42:08.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:08 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/1434043189' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:09.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:09 smithi145 ceph-mon[82523]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:42:09.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:09 smithi145 ceph-mon[82523]: from='client.54520 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:42:09.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:09 smithi145 ceph-mon[82523]: from='client.44589 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:42:09.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:09 smithi145 ceph-mon[88162]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:42:09.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:09 smithi145 ceph-mon[88162]: from='client.54520 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:42:09.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:09 smithi145 ceph-mon[88162]: from='client.44589 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:42:09.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:09 smithi172 ceph-mon[65307]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:42:09.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:09 smithi172 ceph-mon[65307]: from='client.54520 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:42:09.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:09 smithi172 ceph-mon[65307]: from='client.44589 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:42:10.573 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:10 smithi145 ceph-mon[88162]: from='client.54538 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:42:10.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:10 smithi145 ceph-mon[82523]: from='client.54538 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:42:10.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:10 smithi172 ceph-mon[65307]: from='client.54538 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:42:11.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:11 smithi145 ceph-mon[88162]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:42:11.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:11 smithi145 ceph-mon[82523]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:42:11.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:11 smithi172 ceph-mon[65307]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:42:12.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:12 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:42:12.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:12 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:42:12.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:12 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:42:13.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:13 smithi145 ceph-mon[88162]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:42:13.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:13 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T15:42:13.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:13 smithi145 ceph-mon[82523]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:42:13.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:13 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T15:42:13.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:13 smithi172 ceph-mon[65307]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:42:13.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:13 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T15:42:14.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:14 smithi145 ceph-mon[88162]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T15:42:14.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:14 smithi145 ceph-mon[88162]: Upgrade: osd.7 is safe to restart 2024-04-22T15:42:14.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:14 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:14.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:14 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T15:42:14.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:14 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:42:14.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:14 smithi145 ceph-mon[82523]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T15:42:14.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:14 smithi145 ceph-mon[82523]: Upgrade: osd.7 is safe to restart 2024-04-22T15:42:14.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:14 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:14.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:14 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T15:42:14.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:14 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:42:14.663 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:14 smithi172 ceph-mon[65307]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-22T15:42:14.663 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:14 smithi172 ceph-mon[65307]: Upgrade: osd.7 is safe to restart 2024-04-22T15:42:14.664 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:14 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:14.664 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:14 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T15:42:14.664 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:14 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:42:15.362 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:15 smithi172 ceph-mon[65307]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:42:15.362 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:15 smithi172 ceph-mon[65307]: Upgrade: Updating osd.7 2024-04-22T15:42:15.362 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:15 smithi172 ceph-mon[65307]: Deploying daemon osd.7 on smithi172 2024-04-22T15:42:15.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:15 smithi145 ceph-mon[82523]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:42:15.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:15 smithi145 ceph-mon[82523]: Upgrade: Updating osd.7 2024-04-22T15:42:15.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:15 smithi145 ceph-mon[82523]: Deploying daemon osd.7 on smithi172 2024-04-22T15:42:15.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:15 smithi145 ceph-mon[88162]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:42:15.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:15 smithi145 ceph-mon[88162]: Upgrade: Updating osd.7 2024-04-22T15:42:15.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:15 smithi145 ceph-mon[88162]: Deploying daemon osd.7 on smithi172 2024-04-22T15:42:15.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:42:15] "GET /metrics HTTP/1.1" 200 38302 "" "Prometheus/2.43.0" 2024-04-22T15:42:17.272 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:17 smithi172 systemd[1]: Stopping Ceph osd.7 for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:42:17.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:17 smithi145 ceph-mon[82523]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:42:17.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:17 smithi145 ceph-mon[82523]: osd.7 marked itself down and dead 2024-04-22T15:42:17.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:17 smithi145 ceph-mon[88162]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:42:17.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:17 smithi145 ceph-mon[88162]: osd.7 marked itself down and dead 2024-04-22T15:42:17.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:17 smithi172 ceph-mon[65307]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:42:17.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:17 smithi172 ceph-mon[65307]: osd.7 marked itself down and dead 2024-04-22T15:42:17.718 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:17 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[34946]: 2024-04-22T15:42:17.268+0000 7f063b205700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:42:17.718 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:17 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[34946]: 2024-04-22T15:42:17.268+0000 7f063b205700 -1 osd.7 143 *** Got signal Terminated *** 2024-04-22T15:42:17.718 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:17 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[34946]: 2024-04-22T15:42:17.268+0000 7f063b205700 -1 osd.7 143 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T15:42:18.131 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:17 smithi172 podman[90042]: 2024-04-22 15:42:17.821066746 +0000 UTC m=+0.671788162 container died cc9e1d9e7916f87a7e4768864d69d80708b83c84084d164944c1e6f991cd057a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7, release=754, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, io.openshift.expose-services=, vcs-type=git, ceph=True, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container) 2024-04-22T15:42:18.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:18 smithi172 ceph-mon[65307]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:42:18.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:18 smithi172 ceph-mon[65307]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-04-22T15:42:18.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:18 smithi172 ceph-mon[65307]: osdmap e144: 8 total, 7 up, 8 in 2024-04-22T15:42:18.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:18 smithi172 ceph-mon[65307]: osdmap e145: 8 total, 7 up, 8 in 2024-04-22T15:42:18.468 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:18 smithi172 podman[90042]: 2024-04-22 15:42:18.227477036 +0000 UTC m=+1.078198460 container cleanup cc9e1d9e7916f87a7e4768864d69d80708b83c84084d164944c1e6f991cd057a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, version=8, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, distribution-scope=public, io.openshift.expose-services=, architecture=x86_64, build-date=2022-05-03T08:36:31.336870) 2024-04-22T15:42:18.468 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:18 smithi172 bash[90042]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7 2024-04-22T15:42:18.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:18 smithi145 ceph-mon[82523]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:42:18.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:18 smithi145 ceph-mon[82523]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-04-22T15:42:18.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:18 smithi145 ceph-mon[82523]: osdmap e144: 8 total, 7 up, 8 in 2024-04-22T15:42:18.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:18 smithi145 ceph-mon[82523]: osdmap e145: 8 total, 7 up, 8 in 2024-04-22T15:42:18.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:18 smithi145 ceph-mon[88162]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-22T15:42:18.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:18 smithi145 ceph-mon[88162]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-04-22T15:42:18.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:18 smithi145 ceph-mon[88162]: osdmap e144: 8 total, 7 up, 8 in 2024-04-22T15:42:18.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:18 smithi145 ceph-mon[88162]: osdmap e145: 8 total, 7 up, 8 in 2024-04-22T15:42:18.890 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:18 smithi172 podman[90060]: 2024-04-22 15:42:18.56236182 +0000 UTC m=+0.735039832 container remove cc9e1d9e7916f87a7e4768864d69d80708b83c84084d164944c1e6f991cd057a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7, name=centos-stream, version=8, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, architecture=x86_64, io.openshift.expose-services=, maintainer=Guillaume Abrioux , distribution-scope=public, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-22T15:42:19.201 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:18 smithi172 podman[90156]: 2024-04-22 15:42:18.888431231 +0000 UTC m=+0.095079458 container create a01c559197a19557759e144b6adef74ecd66e28cbdd556f14413d989a1422111 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-04-22T15:42:19.201 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:18 smithi172 podman[90156]: 2024-04-22 15:42:18.811872974 +0000 UTC m=+0.018521202 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:42:19.201 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:19 smithi172 podman[90156]: 2024-04-22 15:42:19.10826571 +0000 UTC m=+0.314913939 container init a01c559197a19557759e144b6adef74ecd66e28cbdd556f14413d989a1422111 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T15:42:19.201 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:19 smithi172 podman[90156]: 2024-04-22 15:42:19.111422286 +0000 UTC m=+0.318070524 container start a01c559197a19557759e144b6adef74ecd66e28cbdd556f14413d989a1422111 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS) 2024-04-22T15:42:19.201 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:19 smithi172 podman[90156]: 2024-04-22 15:42:19.199250417 +0000 UTC m=+0.405898654 container attach a01c559197a19557759e144b6adef74ecd66e28cbdd556f14413d989a1422111 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T15:42:19.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:19 smithi172 ceph-mon[65307]: pgmap v259: 161 pgs: 13 peering, 20 stale+active+clean, 128 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:42:19.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:19 smithi172 ceph-mon[65307]: Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2024-04-22T15:42:19.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:19 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:19.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:19 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:42:19.468 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:19 smithi172 podman[90156]: 2024-04-22 15:42:19.408328878 +0000 UTC m=+0.614977107 container died a01c559197a19557759e144b6adef74ecd66e28cbdd556f14413d989a1422111 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T15:42:19.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:19 smithi145 ceph-mon[82523]: pgmap v259: 161 pgs: 13 peering, 20 stale+active+clean, 128 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:42:19.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:19 smithi145 ceph-mon[82523]: Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2024-04-22T15:42:19.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:19 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:19.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:19 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:42:19.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:19 smithi145 ceph-mon[88162]: pgmap v259: 161 pgs: 13 peering, 20 stale+active+clean, 128 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:42:19.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:19 smithi145 ceph-mon[88162]: Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2024-04-22T15:42:19.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:19 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:19.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:19 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:42:20.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:19 smithi172 podman[90181]: 2024-04-22 15:42:19.877645614 +0000 UTC m=+0.463840535 container remove a01c559197a19557759e144b6adef74ecd66e28cbdd556f14413d989a1422111 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7-deactivate, ceph=True, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:42:20.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:19 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.7.service: Deactivated successfully. 2024-04-22T15:42:20.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:19 smithi172 systemd[1]: Stopped Ceph osd.7 for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:42:20.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:19 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.7.service: Consumed 17.914s CPU time. 2024-04-22T15:42:20.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:20 smithi172 systemd[1]: Starting Ceph osd.7 for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:42:20.608 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:20 smithi172 ceph-mon[65307]: pgmap v261: 161 pgs: 1 active+undersized+degraded, 30 peering, 15 stale+active+clean, 6 active+undersized, 109 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1/627 objects degraded (0.159%) 2024-04-22T15:42:20.608 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:20 smithi172 ceph-mon[65307]: Health check failed: Degraded data redundancy: 1/627 objects degraded (0.159%), 1 pg degraded (PG_DEGRADED) 2024-04-22T15:42:20.608 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:20 smithi172 podman[90289]: 2024-04-22 15:42:20.228110907 +0000 UTC m=+0.018666591 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:42:20.608 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:20 smithi172 podman[90289]: 2024-04-22 15:42:20.345408863 +0000 UTC m=+0.135964580 container create 5da77a753495d708dd2e0c99e3cab30677c76da9f94b46128c6d8ee8e04439b6 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7-activate, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-04-22T15:42:20.608 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:20 smithi172 podman[90289]: 2024-04-22 15:42:20.531377398 +0000 UTC m=+0.321933072 container init 5da77a753495d708dd2e0c99e3cab30677c76da9f94b46128c6d8ee8e04439b6 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7-activate, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416) 2024-04-22T15:42:20.608 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:20 smithi172 podman[90289]: 2024-04-22 15:42:20.534512033 +0000 UTC m=+0.325067722 container start 5da77a753495d708dd2e0c99e3cab30677c76da9f94b46128c6d8ee8e04439b6 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7-activate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T15:42:20.609 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:20 smithi172 podman[90289]: 2024-04-22 15:42:20.606145011 +0000 UTC m=+0.396700706 container attach 5da77a753495d708dd2e0c99e3cab30677c76da9f94b46128c6d8ee8e04439b6 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7-activate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-04-22T15:42:20.770 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:20 smithi145 ceph-mon[82523]: pgmap v261: 161 pgs: 1 active+undersized+degraded, 30 peering, 15 stale+active+clean, 6 active+undersized, 109 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1/627 objects degraded (0.159%) 2024-04-22T15:42:20.770 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:20 smithi145 ceph-mon[82523]: Health check failed: Degraded data redundancy: 1/627 objects degraded (0.159%), 1 pg degraded (PG_DEGRADED) 2024-04-22T15:42:20.770 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:20 smithi145 ceph-mon[88162]: pgmap v261: 161 pgs: 1 active+undersized+degraded, 30 peering, 15 stale+active+clean, 6 active+undersized, 109 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1/627 objects degraded (0.159%) 2024-04-22T15:42:20.770 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:20 smithi145 ceph-mon[88162]: Health check failed: Degraded data redundancy: 1/627 objects degraded (0.159%), 1 pg degraded (PG_DEGRADED) 2024-04-22T15:42:21.346 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:21 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7-activate[90305]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:42:21.346 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:21 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7-activate[90305]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:42:21.346 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:21 smithi172 bash[90289]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:42:21.346 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:21 smithi172 bash[90289]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-22T15:42:21.346 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:21 smithi172 bash[90289]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-22T15:42:21.346 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:21 smithi172 bash[90289]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-22T15:42:21.346 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:21 smithi172 bash[90289]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-22T15:42:21.346 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:21 smithi172 bash[90289]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-22T15:42:21.346 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:21 smithi172 bash[90289]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-04-22T15:42:21.347 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:21 smithi172 bash[90289]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-22T15:42:21.347 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:21 smithi172 bash[90289]: --> ceph-volume raw activate successful for osd ID: 7 2024-04-22T15:42:21.347 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:21 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7-activate[90305]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-22T15:42:21.347 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:21 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7-activate[90305]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-22T15:42:21.347 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:21 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7-activate[90305]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-22T15:42:21.347 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:21 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7-activate[90305]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-22T15:42:21.347 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:21 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7-activate[90305]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-04-22T15:42:21.347 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:21 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7-activate[90305]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-22T15:42:21.347 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:21 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7-activate[90305]: --> ceph-volume raw activate successful for osd ID: 7 2024-04-22T15:42:21.347 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:21 smithi172 podman[90289]: 2024-04-22 15:42:21.040340541 +0000 UTC m=+0.830896216 container died 5da77a753495d708dd2e0c99e3cab30677c76da9f94b46128c6d8ee8e04439b6 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7-activate, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:42:21.680 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:21 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:42:21.681 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:21 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:42:21.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:21 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:42:21.717 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:21 smithi172 podman[90438]: 2024-04-22 15:42:21.576820019 +0000 UTC m=+0.526370947 container remove 5da77a753495d708dd2e0c99e3cab30677c76da9f94b46128c6d8ee8e04439b6 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7-activate, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-22T15:42:22.100 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:21 smithi172 podman[90503]: 2024-04-22 15:42:21.69544115 +0000 UTC m=+0.018358710 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:42:22.100 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:21 smithi172 podman[90503]: 2024-04-22 15:42:21.794432477 +0000 UTC m=+0.117350024 container create 951a8da56333a1527aed1fd5682435f1b4c4856ab74ba527d37a9c99c2f373cf (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-04-22T15:42:22.100 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:22 smithi172 podman[90503]: 2024-04-22 15:42:22.005570976 +0000 UTC m=+0.328488523 container init 951a8da56333a1527aed1fd5682435f1b4c4856ab74ba527d37a9c99c2f373cf (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux ) 2024-04-22T15:42:22.100 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:22 smithi172 podman[90503]: 2024-04-22 15:42:22.008781458 +0000 UTC m=+0.331699006 container start 951a8da56333a1527aed1fd5682435f1b4c4856ab74ba527d37a9c99c2f373cf (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T15:42:22.100 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:22 smithi172 ceph-osd[90523]: -- 172.21.15.172:0/3212849198 <== mon.2 v2:172.21.15.172:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56230324f680 con 0x562303219000 2024-04-22T15:42:22.100 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:22 smithi172 bash[90503]: 951a8da56333a1527aed1fd5682435f1b4c4856ab74ba527d37a9c99c2f373cf 2024-04-22T15:42:22.467 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:22 smithi172 systemd[1]: Started Ceph osd.7 for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:42:22.945 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:22 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:42:22.559+0000 7f5e2f37c7c0 -1 Falling back to public interface 2024-04-22T15:42:23.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:23 smithi172 ceph-mon[65307]: pgmap v262: 161 pgs: 14 active+undersized+degraded, 30 peering, 25 active+undersized, 92 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 51/627 objects degraded (8.134%) 2024-04-22T15:42:23.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:23 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:23.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:23 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:23.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:23 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:42:23.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:23 smithi145 ceph-mon[82523]: pgmap v262: 161 pgs: 14 active+undersized+degraded, 30 peering, 25 active+undersized, 92 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 51/627 objects degraded (8.134%) 2024-04-22T15:42:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:23 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:23 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:23.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:23 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:42:23.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:23 smithi145 ceph-mon[88162]: pgmap v262: 161 pgs: 14 active+undersized+degraded, 30 peering, 25 active+undersized, 92 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 51/627 objects degraded (8.134%) 2024-04-22T15:42:23.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:23 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:23.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:23 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:23.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:23 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:42:24.458 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:24 smithi172 ceph-mon[65307]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 4 pgs peering) 2024-04-22T15:42:24.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:24 smithi145 ceph-mon[82523]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 4 pgs peering) 2024-04-22T15:42:24.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:24 smithi145 ceph-mon[88162]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 4 pgs peering) 2024-04-22T15:42:25.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:25 smithi172 ceph-mon[65307]: pgmap v263: 161 pgs: 26 active+undersized+degraded, 43 active+undersized, 92 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 82/627 objects degraded (13.078%) 2024-04-22T15:42:25.467 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:25 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:42:25.299+0000 7f5e2f37c7c0 -1 osd.7 0 read_superblock omap replica is missing. 2024-04-22T15:42:25.468 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:25 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:42:25.327+0000 7f5e2f37c7c0 -1 osd.7 143 log_to_monitors true 2024-04-22T15:42:25.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:25 smithi145 ceph-mon[82523]: pgmap v263: 161 pgs: 26 active+undersized+degraded, 43 active+undersized, 92 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 82/627 objects degraded (13.078%) 2024-04-22T15:42:25.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:25 smithi145 ceph-mon[88162]: pgmap v263: 161 pgs: 26 active+undersized+degraded, 43 active+undersized, 92 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 82/627 objects degraded (13.078%) 2024-04-22T15:42:25.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:42:25] "GET /metrics HTTP/1.1" 200 38378 "" "Prometheus/2.43.0" 2024-04-22T15:42:26.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:26 smithi172 ceph-mon[65307]: from='osd.7 [v2:172.21.15.172:6824/1852829315,v1:172.21.15.172:6825/1852829315]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T15:42:26.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:26 smithi172 ceph-mon[65307]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T15:42:26.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:26 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:26.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:26 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:26.468 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:42:26 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:42:26.229+0000 7f5e207d4700 -1 osd.7 143 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T15:42:26.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:26 smithi145 ceph-mon[82523]: from='osd.7 [v2:172.21.15.172:6824/1852829315,v1:172.21.15.172:6825/1852829315]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T15:42:26.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:26 smithi145 ceph-mon[82523]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T15:42:26.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:26 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:26.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:26 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:26.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:26 smithi145 ceph-mon[88162]: from='osd.7 [v2:172.21.15.172:6824/1852829315,v1:172.21.15.172:6825/1852829315]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T15:42:26.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:26 smithi145 ceph-mon[88162]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T15:42:26.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:26 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:26.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:26 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:27.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:27 smithi145 ceph-mon[82523]: pgmap v264: 161 pgs: 26 active+undersized+degraded, 43 active+undersized, 92 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 82/627 objects degraded (13.078%) 2024-04-22T15:42:27.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:27 smithi145 ceph-mon[82523]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-22T15:42:27.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:27 smithi145 ceph-mon[82523]: osdmap e146: 8 total, 7 up, 8 in 2024-04-22T15:42:27.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:27 smithi145 ceph-mon[82523]: from='osd.7 [v2:172.21.15.172:6824/1852829315,v1:172.21.15.172:6825/1852829315]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:42:27.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:27 smithi145 ceph-mon[82523]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:42:27.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:27 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:27.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:27 smithi145 ceph-mon[88162]: pgmap v264: 161 pgs: 26 active+undersized+degraded, 43 active+undersized, 92 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 82/627 objects degraded (13.078%) 2024-04-22T15:42:27.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:27 smithi145 ceph-mon[88162]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-22T15:42:27.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:27 smithi145 ceph-mon[88162]: osdmap e146: 8 total, 7 up, 8 in 2024-04-22T15:42:27.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:27 smithi145 ceph-mon[88162]: from='osd.7 [v2:172.21.15.172:6824/1852829315,v1:172.21.15.172:6825/1852829315]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:42:27.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:27 smithi145 ceph-mon[88162]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:42:27.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:27 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:27.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:27 smithi172 ceph-mon[65307]: pgmap v264: 161 pgs: 26 active+undersized+degraded, 43 active+undersized, 92 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 82/627 objects degraded (13.078%) 2024-04-22T15:42:27.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:27 smithi172 ceph-mon[65307]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-22T15:42:27.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:27 smithi172 ceph-mon[65307]: osdmap e146: 8 total, 7 up, 8 in 2024-04-22T15:42:27.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:27 smithi172 ceph-mon[65307]: from='osd.7 [v2:172.21.15.172:6824/1852829315,v1:172.21.15.172:6825/1852829315]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:42:27.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:27 smithi172 ceph-mon[65307]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi172", "root=default"]}]: dispatch 2024-04-22T15:42:27.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:27 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:28.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:28 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:28.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:28 smithi172 ceph-mon[65307]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:42:28.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:28 smithi172 ceph-mon[65307]: osd.7 [v2:172.21.15.172:6824/1852829315,v1:172.21.15.172:6825/1852829315] boot 2024-04-22T15:42:28.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:28 smithi172 ceph-mon[65307]: osdmap e147: 8 total, 8 up, 8 in 2024-04-22T15:42:28.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:28 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:42:28.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:28 smithi172 ceph-mon[65307]: osdmap e148: 8 total, 8 up, 8 in 2024-04-22T15:42:28.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:28 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:28.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:28 smithi145 ceph-mon[82523]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:42:28.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:28 smithi145 ceph-mon[82523]: osd.7 [v2:172.21.15.172:6824/1852829315,v1:172.21.15.172:6825/1852829315] boot 2024-04-22T15:42:28.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:28 smithi145 ceph-mon[82523]: osdmap e147: 8 total, 8 up, 8 in 2024-04-22T15:42:28.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:28 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:42:28.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:28 smithi145 ceph-mon[82523]: osdmap e148: 8 total, 8 up, 8 in 2024-04-22T15:42:28.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:28 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:28.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:28 smithi145 ceph-mon[88162]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-22T15:42:28.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:28 smithi145 ceph-mon[88162]: osd.7 [v2:172.21.15.172:6824/1852829315,v1:172.21.15.172:6825/1852829315] boot 2024-04-22T15:42:28.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:28 smithi145 ceph-mon[88162]: osdmap e147: 8 total, 8 up, 8 in 2024-04-22T15:42:28.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:28 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T15:42:28.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:28 smithi145 ceph-mon[88162]: osdmap e148: 8 total, 8 up, 8 in 2024-04-22T15:42:29.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:29 smithi145 ceph-mon[82523]: pgmap v267: 161 pgs: 10 peering, 23 active+undersized+degraded, 36 active+undersized, 92 active+clean; 585 KiB data, 240 MiB used, 715 GiB / 715 GiB avail; 75/627 objects degraded (11.962%) 2024-04-22T15:42:29.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:29 smithi145 ceph-mon[82523]: Health check update: Degraded data redundancy: 75/627 objects degraded (11.962%), 23 pgs degraded (PG_DEGRADED) 2024-04-22T15:42:29.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:29 smithi145 ceph-mon[88162]: pgmap v267: 161 pgs: 10 peering, 23 active+undersized+degraded, 36 active+undersized, 92 active+clean; 585 KiB data, 240 MiB used, 715 GiB / 715 GiB avail; 75/627 objects degraded (11.962%) 2024-04-22T15:42:29.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:29 smithi145 ceph-mon[88162]: Health check update: Degraded data redundancy: 75/627 objects degraded (11.962%), 23 pgs degraded (PG_DEGRADED) 2024-04-22T15:42:29.698 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:29 smithi172 ceph-mon[65307]: pgmap v267: 161 pgs: 10 peering, 23 active+undersized+degraded, 36 active+undersized, 92 active+clean; 585 KiB data, 240 MiB used, 715 GiB / 715 GiB avail; 75/627 objects degraded (11.962%) 2024-04-22T15:42:29.699 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:29 smithi172 ceph-mon[65307]: Health check update: Degraded data redundancy: 75/627 objects degraded (11.962%), 23 pgs degraded (PG_DEGRADED) 2024-04-22T15:42:31.301 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:31 smithi145 ceph-mon[82523]: pgmap v269: 161 pgs: 12 activating, 13 peering, 17 active+undersized+degraded, 23 active+undersized, 96 active+clean; 585 KiB data, 240 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 59/627 objects degraded (9.410%) 2024-04-22T15:42:31.301 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:31 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:31.301 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:31 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:31.301 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:31 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:42:31.301 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:31 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:42:31.301 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:31 smithi145 ceph-mon[82523]: pgmap v270: 161 pgs: 12 activating, 13 peering, 14 active+undersized+degraded, 22 active+undersized, 100 active+clean; 585 KiB data, 244 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 47/627 objects degraded (7.496%) 2024-04-22T15:42:31.301 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:31 smithi145 ceph-mon[82523]: pgmap v271: 161 pgs: 12 activating, 13 peering, 14 active+undersized+degraded, 22 active+undersized, 100 active+clean; 585 KiB data, 244 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 47/627 objects degraded (7.496%) 2024-04-22T15:42:31.301 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:31 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:31.301 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:31 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:42:31.301 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:31 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:42:31.302 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:31 smithi145 ceph-mon[88162]: pgmap v269: 161 pgs: 12 activating, 13 peering, 17 active+undersized+degraded, 23 active+undersized, 96 active+clean; 585 KiB data, 240 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 59/627 objects degraded (9.410%) 2024-04-22T15:42:31.302 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:31 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:31.302 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:31 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:31.302 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:31 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:42:31.302 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:31 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:42:31.302 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:31 smithi145 ceph-mon[88162]: pgmap v270: 161 pgs: 12 activating, 13 peering, 14 active+undersized+degraded, 22 active+undersized, 100 active+clean; 585 KiB data, 244 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 47/627 objects degraded (7.496%) 2024-04-22T15:42:31.302 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:31 smithi145 ceph-mon[88162]: pgmap v271: 161 pgs: 12 activating, 13 peering, 14 active+undersized+degraded, 22 active+undersized, 100 active+clean; 585 KiB data, 244 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 47/627 objects degraded (7.496%) 2024-04-22T15:42:31.302 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:31 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:31.302 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:31 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:42:31.302 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:31 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:42:31.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:31 smithi172 ceph-mon[65307]: pgmap v269: 161 pgs: 12 activating, 13 peering, 17 active+undersized+degraded, 23 active+undersized, 96 active+clean; 585 KiB data, 240 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 59/627 objects degraded (9.410%) 2024-04-22T15:42:31.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:31 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:31.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:31 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:31 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:42:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:31 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:42:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:31 smithi172 ceph-mon[65307]: pgmap v270: 161 pgs: 12 activating, 13 peering, 14 active+undersized+degraded, 22 active+undersized, 100 active+clean; 585 KiB data, 244 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 47/627 objects degraded (7.496%) 2024-04-22T15:42:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:31 smithi172 ceph-mon[65307]: pgmap v271: 161 pgs: 12 activating, 13 peering, 14 active+undersized+degraded, 22 active+undersized, 100 active+clean; 585 KiB data, 244 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 47/627 objects degraded (7.496%) 2024-04-22T15:42:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:31 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:31 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:42:31.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:31 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:42:32.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-mon[88162]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:42:32.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-mon[88162]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:42:32.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:42:32.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-mon[82523]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:42:32.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-mon[82523]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:42:32.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:42:32.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:42:32.299+0000 7f17b18ae700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:42:32.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:42:32.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:42:32.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:42:32.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:42:32.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:32.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:32.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:42:32.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:42:32.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:42:32.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:42:32.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:42:32.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:42:32.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:42:32.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:42:32.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:42:32.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:42:32.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:42:32.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:42:32.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:32.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:32.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:42:32.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:42:32.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:42:32.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:42:32.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:42:32.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:42:32.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:42:32.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:42:32.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:42:32.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:42:32.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:42:32.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:42:32.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:42:32.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:42:32.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:42:32.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:42:32.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:42:32.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:42:32.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:42:32.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:42:32.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:42:32.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:42:32.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:32.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:32.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:42:32.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:42:32.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:42:32.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:42:32.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:42:32.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:42:32.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:42:32.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:42:32.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:42:32.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:42:32.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:42:32.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:42:32.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:32.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:32 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:32.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:32 smithi172 ceph-mon[65307]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:42:32.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:32 smithi172 ceph-mon[65307]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:42:32.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:32 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:42:33.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[88162]: pgmap v272: 161 pgs: 12 activating, 13 peering, 136 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1006 B/s rd, 0 op/s 2024-04-22T15:42:33.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[88162]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:42:33.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:42:33.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:42:33.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:42:33.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:42:33.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:33.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:33.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:42:33.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:42:33.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:42:33.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:42:33.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:42:33.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:42:33.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:42:33.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:42:33.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:42:33.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:42:33.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:42:33.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:42:33.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:33.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:33.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:42:33.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:42:33.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:42:33.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:42:33.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:42:33.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:42:33.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:42:33.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:42:33.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:42:33.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:42:33.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:42:33.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:42:33.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:42:33.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:42:33.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:42:33.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:42:33.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:42:33.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:42:33.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:42:33.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:42:33.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:42:33.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:42:33.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:33.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:33.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:42:33.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:42:33.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:42:33.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:42:33.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:42:33.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:42:33.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:42:33.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:42:33.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:42:33.626 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[82523]: pgmap v272: 161 pgs: 12 activating, 13 peering, 136 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1006 B/s rd, 0 op/s 2024-04-22T15:42:33.627 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[82523]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:42:33.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:42:33.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:42:33.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:42:33.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:42:33.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:33.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:33.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:42:33.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:42:33.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:42:33.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:42:33.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:42:33.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:42:33.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:42:33.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:42:33.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:42:33.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:42:33.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:42:33.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:42:33.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:33.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:33.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:42:33.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:42:33.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:42:33.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:42:33.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:42:33.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:42:33.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:42:33.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:42:33.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:42:33.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:42:33.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:42:33.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:42:33.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:42:33.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:42:33.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:42:33.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:42:33.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:42:33.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:42:33.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:42:33.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:42:33.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:42:33.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:42:33.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:33.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:33.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:42:33.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:42:33.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:42:33.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:42:33.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:42:33.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:42:33.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:42:33.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:42:33.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:42:33.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:42:33.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:42:33.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:42:33.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:33.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:33.630 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:42:33.630 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:33.630 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:33.630 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:33.630 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:33.631 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all osd 2024-04-22T15:42:33.631 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:33.631 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-22T15:42:33.631 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-04-22T15:42:33.631 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-22T15:42:33.631 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-04-22T15:42:33.631 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-22T15:42:33.631 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-04-22T15:42:33.631 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-22T15:42:33.631 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-04-22T15:42:33.631 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-22T15:42:33.631 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-04-22T15:42:33.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:42:33.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:42:33.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:42:33.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:33.632 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:33.632 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:42:33.632 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:33.632 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:33.632 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:33.632 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:33.632 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all osd 2024-04-22T15:42:33.632 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:33.632 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-22T15:42:33.632 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-04-22T15:42:33.632 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-22T15:42:33.632 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-04-22T15:42:33.633 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-22T15:42:33.633 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-04-22T15:42:33.633 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-22T15:42:33.633 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-04-22T15:42:33.633 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-22T15:42:33.633 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-04-22T15:42:33.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-22T15:42:33.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-04-22T15:42:33.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-22T15:42:33.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-04-22T15:42:33.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-22T15:42:33.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-04-22T15:42:33.634 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[82523]: Upgrade: Setting require_osd_release to 19 squid 2024-04-22T15:42:33.634 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-22T15:42:33.634 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-22T15:42:33.634 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-04-22T15:42:33.634 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-22T15:42:33.634 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-04-22T15:42:33.634 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-22T15:42:33.634 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-04-22T15:42:33.634 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[88162]: Upgrade: Setting require_osd_release to 19 squid 2024-04-22T15:42:33.634 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:33 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-22T15:42:33.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:33 smithi172 ceph-mon[65307]: pgmap v272: 161 pgs: 12 activating, 13 peering, 136 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 1006 B/s rd, 0 op/s 2024-04-22T15:42:33.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:33 smithi172 ceph-mon[65307]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:42:33.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:42:33.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:42:33.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:42:33.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:42:33.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:33.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:33.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:42:33.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:42:33.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:42:33.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:42:33.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:42:33.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:42:33.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:42:33.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:42:33.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:42:33.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:42:33.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:42:33.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:42:33.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:33.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:33.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:42:33.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:42:33.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:42:33.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:42:33.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:42:33.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:42:33.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:42:33.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:42:33.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:42:33.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:42:33.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:42:33.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:42:33.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:42:33.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:42:33.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:42:33.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:42:33.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:42:33.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:42:33.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:42:33.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:42:33.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:42:33.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:42:33.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:33.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:33.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:42:33.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:42:33.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:42:33.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:42:33.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:42:33.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:42:33.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:42:33.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:42:33.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:42:33.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:42:33.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:42:33.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:42:33.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:33.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:33.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:33 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:42:33.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:33 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:33.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:33 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:33.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:33 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:33.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:33 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:33.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:33 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all osd 2024-04-22T15:42:33.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:33 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:33.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:33 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-22T15:42:33.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:33 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-04-22T15:42:33.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:33 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-22T15:42:33.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:33 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-04-22T15:42:33.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:33 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-22T15:42:33.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:33 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-04-22T15:42:33.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:33 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-22T15:42:33.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:33 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-04-22T15:42:33.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:33 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-22T15:42:33.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:33 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-04-22T15:42:33.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:33 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-22T15:42:33.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:33 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-04-22T15:42:33.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:33 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-22T15:42:33.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:33 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-04-22T15:42:33.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:33 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-22T15:42:33.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:33 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-04-22T15:42:33.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:33 smithi172 ceph-mon[65307]: Upgrade: Setting require_osd_release to 19 squid 2024-04-22T15:42:33.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:33 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-22T15:42:34.451 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 47/627 objects degraded (7.496%), 14 pgs degraded) 2024-04-22T15:42:34.451 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-04-22T15:42:34.451 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-04-22T15:42:34.451 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: osdmap e149: 8 total, 8 up, 8 in 2024-04-22T15:42:34.451 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.451 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all mds 2024-04-22T15:42:34.451 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:34.451 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.452 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T15:42:34.452 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:34.452 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.452 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.452 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T15:42:34.452 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:34.452 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.452 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:34.452 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.452 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:34.452 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.452 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.452 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.452 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.452 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.452 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.453 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T15:42:34.453 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T15:42:34.453 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:42:34.453 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T15:42:34.453 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T15:42:34.453 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 47/627 objects degraded (7.496%), 14 pgs degraded) 2024-04-22T15:42:34.453 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-04-22T15:42:34.453 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-04-22T15:42:34.453 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: osdmap e149: 8 total, 8 up, 8 in 2024-04-22T15:42:34.453 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.453 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all mds 2024-04-22T15:42:34.453 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:34.454 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.454 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T15:42:34.454 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:34.454 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.454 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.454 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T15:42:34.454 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:34.454 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.454 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:34.454 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.454 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:34.454 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.454 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.455 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.455 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.455 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.455 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.455 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T15:42:34.455 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T15:42:34.455 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:42:34.455 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T15:42:34.455 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T15:42:34.455 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T15:42:34.456 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T15:42:34.456 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-22T15:42:34.456 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T15:42:34.456 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T15:42:34.456 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T15:42:34.456 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T15:42:34.456 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T15:42:34.456 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:42:34.456 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T15:42:34.456 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T15:42:34.456 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T15:42:34.457 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T15:42:34.457 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T15:42:34.457 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T15:42:34.457 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T15:42:34.457 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:42:34.457 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:42:34.457 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:42:34.457 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:42:34.457 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:42:34.457 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:42:34.457 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T15:42:34.457 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T15:42:34.457 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:42:34.457 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:42:34.458 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:42:34.458 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:34.458 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:42:34.458 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:42:34.458 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T15:42:34.458 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T15:42:34.458 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-22T15:42:34.458 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T15:42:34.461 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T15:42:34.461 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T15:42:34.461 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T15:42:34.461 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T15:42:34.461 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:42:34.461 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T15:42:34.462 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T15:42:34.462 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T15:42:34.462 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T15:42:34.462 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T15:42:34.462 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T15:42:34.462 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T15:42:34.462 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:42:34.462 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:42:34.462 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:42:34.462 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:42:34.462 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:42:34.462 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:42:34.462 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T15:42:34.462 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T15:42:34.463 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:42:34.463 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:42:34.463 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:42:34.463 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:34.463 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:42:34.463 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:42:34.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 47/627 objects degraded (7.496%), 14 pgs degraded) 2024-04-22T15:42:34.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-04-22T15:42:34.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-04-22T15:42:34.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: osdmap e149: 8 total, 8 up, 8 in 2024-04-22T15:42:34.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all mds 2024-04-22T15:42:34.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:34.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T15:42:34.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:34.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T15:42:34.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:34.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:34.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:34.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:34.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T15:42:34.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T15:42:34.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:42:34.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T15:42:34.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T15:42:34.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T15:42:34.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T15:42:34.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-22T15:42:34.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T15:42:34.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T15:42:34.720 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T15:42:34.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T15:42:34.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T15:42:34.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:42:34.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T15:42:34.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T15:42:34.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T15:42:34.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T15:42:34.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T15:42:34.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T15:42:34.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T15:42:34.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:42:34.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:42:34.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:42:34.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:42:34.721 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:42:34.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:42:34.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T15:42:34.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T15:42:34.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:42:34.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:42:34.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:42:34.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:34.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:42:34.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:42:35.467 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all nfs 2024-04-22T15:42:35.467 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all nvmeof 2024-04-22T15:42:35.467 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-mon[88162]: Upgrade: Finalizing container_image settings 2024-04-22T15:42:35.467 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-mon[88162]: Upgrade: Complete! 2024-04-22T15:42:35.467 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-mon[88162]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s 2024-04-22T15:42:35.468 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-mon[88162]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:42:35.468 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-mon[88162]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:42:35.468 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-mon[88162]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:42:35.468 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:35.468 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:42:35.468 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:35.468 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all nfs 2024-04-22T15:42:35.468 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all nvmeof 2024-04-22T15:42:35.468 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-mon[82523]: Upgrade: Finalizing container_image settings 2024-04-22T15:42:35.468 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-mon[82523]: Upgrade: Complete! 2024-04-22T15:42:35.468 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-mon[82523]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s 2024-04-22T15:42:35.469 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-mon[82523]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:42:35.469 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-mon[82523]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:42:35.469 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-mon[82523]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:42:35.469 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:35.469 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:42:35.469 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:35.469 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:42:35] "GET /metrics HTTP/1.1" 200 38386 "" "Prometheus/2.43.0" 2024-04-22T15:42:35.469 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:42:35.462+0000 7f17b18ae700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:42:35.470 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:42:35.470 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:42:35.470 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:42:35.470 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:42:35.470 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:35.470 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:35.470 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:42:35.470 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:42:35.470 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:42:35.470 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:42:35.470 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:42:35.470 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:42:35.470 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:42:35.470 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:42:35.471 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:42:35.471 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:42:35.471 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:42:35.471 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:42:35.471 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:35.471 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:35.471 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:42:35.471 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:42:35.471 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:42:35.471 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:42:35.471 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:42:35.471 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:42:35.471 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:42:35.471 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:42:35.472 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:42:35.472 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:42:35.472 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:42:35.472 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:42:35.472 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:42:35.472 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:42:35.472 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:42:35.472 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:42:35.472 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:42:35.472 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:42:35.472 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:42:35.472 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:42:35.472 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:42:35.472 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:42:35.472 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:35.473 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:35.473 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:42:35.473 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:42:35.473 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:42:35.473 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:42:35.473 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:42:35.473 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:42:35.473 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:42:35.473 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:42:35.473 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:42:35.473 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:42:35.473 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:42:35.473 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:42:35.473 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:35.473 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:35.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:35 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all nfs 2024-04-22T15:42:35.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:35 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all nvmeof 2024-04-22T15:42:35.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:35 smithi172 ceph-mon[65307]: Upgrade: Finalizing container_image settings 2024-04-22T15:42:35.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:35 smithi172 ceph-mon[65307]: Upgrade: Complete! 2024-04-22T15:42:35.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:35 smithi172 ceph-mon[65307]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s 2024-04-22T15:42:35.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:35 smithi172 ceph-mon[65307]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:42:35.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:35 smithi172 ceph-mon[65307]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:42:35.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:35 smithi172 ceph-mon[65307]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:42:35.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:35.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:42:35.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:35 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:36.434 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:36 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:42:36.434 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:36 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:42:36.434 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:36 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:42:36.434 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:36 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:36.434 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:36 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:42:36.434 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:36 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:42:36.434 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:36 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:42:36.434 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:36 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:42:36.434 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:36 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:42:36.434 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:36 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:36.435 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:36 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:42:36.435 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:36 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:42:36.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:36 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:42:36.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:36 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:42:36.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:36 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:42:36.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:36 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:36.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:36 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:42:36.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:36 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:42:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-mon[88162]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-04-22T15:42:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-mon[88162]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:42:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:42:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:42:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:42:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:42:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:42:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:42:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:42:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:42:37.623 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:42:37.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:42:37.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:42:37.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:42:37.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:42:37.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:42:37.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:42:37.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:42:37.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:37.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:37.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:42:37.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:42:37.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:42:37.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:42:37.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:42:37.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:42:37.624 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:42:37.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:42:37.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:42:37.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:42:37.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:42:37.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:42:37.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:42:37.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:42:37.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:42:37.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:42:37.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:42:37.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:42:37.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:42:37.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:42:37.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:42:37.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:42:37.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:37.625 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:37.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:42:37.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:42:37.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:42:37.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:42:37.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:42:37.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:42:37.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:42:37.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:42:37.626 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:42:37.627 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-mon[82523]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-04-22T15:42:37.627 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-mon[82523]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:42:37.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:42:37.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:42:37.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:42:37.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:42:37.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:37.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:37.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:42:37.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:42:37.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:42:37.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:42:37.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:42:37.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:42:37.627 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:42:37.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:42:37.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:42:37.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:42:37.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:42:37.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:42:37.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:37.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:37.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:42:37.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:42:37.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:42:37.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:42:37.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:42:37.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:42:37.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:42:37.628 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:42:37.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:42:37.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:42:37.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:42:37.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:42:37.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:42:37.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:42:37.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:42:37.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:42:37.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:42:37.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:42:37.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:42:37.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:42:37.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:42:37.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:42:37.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:37.629 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:37.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:42:37.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:42:37.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:42:37.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:42:37.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:42:37.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:42:37.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:42:37.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:42:37.630 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:42:37.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:42:37.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:42:37.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:42:37.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:37.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:37.631 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-mon[88162]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 988 B/s rd, 0 op/s 2024-04-22T15:42:37.631 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-mon[88162]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-22T15:42:37.631 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-mon[88162]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:42:37.631 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-mon[88162]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:42:37.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:42:37.632 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:42:37.632 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:42:37.632 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:37.632 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:37.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-mon[82523]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 988 B/s rd, 0 op/s 2024-04-22T15:42:37.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-mon[82523]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-22T15:42:37.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-mon[82523]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:42:37.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-mon[82523]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:42:37.632 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:42:37.445+0000 7f17b18ae700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:42:37.633 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:42:37.633 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:42:37.633 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:42:37.633 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:42:37.633 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:37.633 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:37.633 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:42:37.633 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:42:37.633 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:42:37.633 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:42:37.633 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:42:37.633 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:42:37.633 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:42:37.633 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:42:37.634 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:42:37.634 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:42:37.634 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:42:37.634 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:42:37.634 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:37.634 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:37.634 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:42:37.634 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:42:37.634 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:42:37.634 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:42:37.634 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:42:37.634 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:42:37.634 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:42:37.635 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:42:37.635 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:42:37.635 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:42:37.635 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:42:37.635 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:42:37.635 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:42:37.635 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:42:37.635 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:42:37.635 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:42:37.635 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:42:37.635 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:42:37.635 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:42:37.635 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:42:37.636 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:42:37.636 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:42:37.636 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:37.636 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:37.636 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:42:37.636 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:42:37.636 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:42:37.636 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:42:37.636 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:42:37.636 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:42:37.636 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:42:37.636 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:42:37.636 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:42:37.636 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:42:37.636 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:42:37.636 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:42:37.637 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:37.637 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:37 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:37.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:37 smithi172 ceph-mon[65307]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-04-22T15:42:37.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:37 smithi172 ceph-mon[65307]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:42:37.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:42:37.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:42:37.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:42:37.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:42:37.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:37.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:37.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:42:37.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:42:37.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:42:37.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:42:37.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:42:37.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:42:37.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:42:37.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:42:37.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:42:37.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:42:37.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:42:37.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:42:37.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:37.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:37.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:42:37.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:42:37.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:42:37.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:42:37.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:42:37.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:42:37.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:42:37.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:42:37.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:42:37.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:42:37.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:42:37.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:42:37.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:42:37.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:42:37.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:42:37.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:42:37.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:42:37.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:42:37.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:42:37.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:42:37.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:42:37.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:42:37.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:37.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:37.723 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:42:37.723 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:42:37.723 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:42:37.723 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:42:37.723 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:42:37.723 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:42:37.723 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:42:37.723 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:42:37.723 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:42:37.724 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:42:37.724 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:42:37.724 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:42:37.724 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:37.724 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:37.724 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:37 smithi172 ceph-mon[65307]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 988 B/s rd, 0 op/s 2024-04-22T15:42:37.724 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:37 smithi172 ceph-mon[65307]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-22T15:42:37.724 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:37 smithi172 ceph-mon[65307]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:42:37.725 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:37 smithi172 ceph-mon[65307]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:42:39.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:39 smithi145 ceph-mon[82523]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:42:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:42:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:42:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:42:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:42:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:42:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:42:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:42:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:42:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:42:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:42:39.623 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:42:39.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:42:39.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:42:39.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:42:39.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:42:39.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:42:39.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:39.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:39.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:42:39.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:42:39.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:42:39.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:42:39.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:42:39.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:42:39.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:42:39.624 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:42:39.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:42:39.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:42:39.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:42:39.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:42:39.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:42:39.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:42:39.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:42:39.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:42:39.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:42:39.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:42:39.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:42:39.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:42:39.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:42:39.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:42:39.625 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:39.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:39.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:42:39.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:42:39.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:42:39.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:42:39.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:42:39.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:42:39.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:42:39.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:42:39.626 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:42:39.627 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:39 smithi145 ceph-mon[88162]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:42:39.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:42:39.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:42:39.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:42:39.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:42:39.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:39.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:39.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:42:39.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:42:39.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:42:39.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:42:39.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:42:39.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:42:39.627 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:42:39.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:42:39.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:42:39.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:42:39.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:42:39.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:42:39.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:39.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:39.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:42:39.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:42:39.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:42:39.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:42:39.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:42:39.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:42:39.628 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:42:39.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:42:39.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:42:39.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:42:39.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:42:39.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:42:39.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:42:39.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:42:39.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:42:39.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:42:39.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:42:39.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:42:39.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:42:39.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:42:39.629 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:42:39.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:42:39.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:39.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:39.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:42:39.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:42:39.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:42:39.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:42:39.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:42:39.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:42:39.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:42:39.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:42:39.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:42:39.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:42:39.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:42:39.630 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:42:39.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:39.631 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:39.631 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:39 smithi145 ceph-mon[88162]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:42:39.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:42:39.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:42:39.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:42:39.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:39.631 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:39.631 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:39 smithi145 ceph-mon[82523]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:42:39.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:39 smithi172 ceph-mon[65307]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:42:39.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:42:39.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:42:39.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:42:39.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:42:39.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:39.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:39.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:42:39.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:42:39.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:42:39.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:42:39.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:42:39.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:42:39.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:42:39.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:42:39.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:42:39.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:42:39.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:42:39.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:42:39.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:39.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:39.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:42:39.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:42:39.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:42:39.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:42:39.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:42:39.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:42:39.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:42:39.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:42:39.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:42:39.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:42:39.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:42:39.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:42:39.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:42:39.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:42:39.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:42:39.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:42:39.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:42:39.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:42:39.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:42:39.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:42:39.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:42:39.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:42:39.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:39.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:39.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:42:39.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:42:39.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:42:39.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:42:39.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:42:39.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:42:39.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:42:39.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:42:39.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:42:39.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:42:39.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:42:39.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:42:39.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:42:39.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:42:39.722 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:39 smithi172 ceph-mon[65307]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:42:39.726 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch ps' 2024-04-22T15:42:40.591 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:40 smithi145 ceph-mon[82523]: from='client.54550 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:42:40.591 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:40 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:40.591 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:40 smithi145 ceph-mon[88162]: from='client.54550 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:42:40.593 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:40 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:40.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:40 smithi172 ceph-mon[65307]: from='client.54550 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:42:40.619 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:40 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:41.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:41 smithi145 ceph-mon[82523]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 850 B/s rd, 0 op/s 2024-04-22T15:42:41.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:41 smithi145 ceph-mon[88162]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 850 B/s rd, 0 op/s 2024-04-22T15:42:41.637 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:41 smithi172 ceph-mon[65307]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 850 B/s rd, 0 op/s 2024-04-22T15:42:41.721 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:42:41.721 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (9m) 2m ago 31m 27.0M - 0.25.0 c8568f914cd2 1c58ccf37ffd 2024-04-22T15:42:41.721 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (9m) 15s ago 31m 79.5M - 9.4.12 2bacad6d85d8 2f825d15ab10 2024-04-22T15:42:41.721 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 unknown 2m ago 30m 60.7M - 3.5 e1d6a67b021e 9704631ffa0e 2024-04-22T15:42:41.722 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443,9283,8765 running (10m) 15s ago 36m 454M - 19.0.0-3223-g5883d494 3d983c26e278 e7500ef71604 2024-04-22T15:42:41.722 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (20m) 2m ago 38m 511M - 19.0.0-3223-g5883d494 3d983c26e278 b2de97566cf4 2024-04-22T15:42:41.722 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (7m) 2m ago 38m 53.7M 2048M 19.0.0-3223-g5883d494 3d983c26e278 7bfdc11120f4 2024-04-22T15:42:41.722 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (7m) 15s ago 36m 51.7M 2048M 19.0.0-3223-g5883d494 3d983c26e278 0a360b08a6d6 2024-04-22T15:42:41.722 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (6m) 2m ago 36m 42.0M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38dcfa677c18 2024-04-22T15:42:41.722 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (10m) 2m ago 32m 15.3M - 1.5.0 0da6a335fe13 8c75c26fec05 2024-04-22T15:42:41.722 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (10m) 15s ago 32m 15.8M - 1.5.0 0da6a335fe13 8fd2949e2eaf 2024-04-22T15:42:41.722 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (4m) 2m ago 36m 51.7M 2992M 19.0.0-3223-g5883d494 3d983c26e278 e4bc06317af0 2024-04-22T15:42:41.722 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (4m) 2m ago 35m 44.0M 2992M 19.0.0-3223-g5883d494 3d983c26e278 d299f622f08d 2024-04-22T15:42:41.722 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (3m) 2m ago 35m 41.7M 2992M 19.0.0-3223-g5883d494 3d983c26e278 6aea4a926fe4 2024-04-22T15:42:41.722 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (2m) 2m ago 34m 15.1M 2992M 19.0.0-3223-g5883d494 3d983c26e278 c0cd2bdc1439 2024-04-22T15:42:41.722 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (2m) 15s ago 34m 47.6M 3248M 19.0.0-3223-g5883d494 3d983c26e278 7ac5776bc07a 2024-04-22T15:42:41.722 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (90s) 15s ago 34m 52.8M 3248M 19.0.0-3223-g5883d494 3d983c26e278 5f803668a17b 2024-04-22T15:42:41.722 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (55s) 15s ago 33m 45.8M 3248M 19.0.0-3223-g5883d494 3d983c26e278 9e049dd4678d 2024-04-22T15:42:41.723 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (20s) 15s ago 33m 12.5M 3248M 19.0.0-3223-g5883d494 3d983c26e278 951a8da56333 2024-04-22T15:42:41.723 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (9m) 15s ago 32m 77.0M - 2.43.0 a07b618ecd1d 51cf4df6e8d4 2024-04-22T15:42:41.723 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (31m) 2m ago 31m 80.2M - 17.2.0 e1d6a67b021e c90a636c947e 2024-04-22T15:42:41.723 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (31m) 15s ago 31m 78.7M - 17.2.0 e1d6a67b021e 20ac46e9bfda 2024-04-22T15:42:42.494 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 1'"'"'' 2024-04-22T15:42:42.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:42 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:42:42.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:42 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:42:42.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:42 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:42:43.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:43 smithi145 ceph-mon[88162]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-04-22T15:42:43.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:43 smithi145 ceph-mon[88162]: from='client.54556 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:42:43.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:43 smithi145 ceph-mon[82523]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-04-22T15:42:43.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:43 smithi145 ceph-mon[82523]: from='client.54556 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:42:43.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:43 smithi172 ceph-mon[65307]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-04-22T15:42:43.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:43 smithi172 ceph-mon[65307]: from='client.54556 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:42:44.599 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:42:45.185 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph versions | jq -e '"'"'.osd | keys'"'"' | grep $sha1' 2024-04-22T15:42:45.493 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:45 smithi145 ceph-mon[88162]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:42:45.493 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:45 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/1309688018' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:45.493 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:45 smithi145 ceph-mon[82523]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:42:45.494 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:45 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/1309688018' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:45.494 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:42:45] "GET /metrics HTTP/1.1" 200 38386 "" "Prometheus/2.43.0" 2024-04-22T15:42:45.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:45 smithi172 ceph-mon[65307]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:42:45.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:45 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/1309688018' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:47.374 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)" 2024-04-22T15:42:47.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:47 smithi145 ceph-mon[82523]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:42:47.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:47 smithi145 ceph-mon[88162]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:42:47.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:47 smithi172 ceph-mon[65307]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:42:48.041 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch upgrade status' 2024-04-22T15:42:48.612 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:48 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/4000121438' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:48.612 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:48 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/4000121438' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:48.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:48 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/4000121438' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:42:49.621 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:49 smithi145 ceph-mon[88162]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:42:49.621 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:49 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:42:49.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:49 smithi145 ceph-mon[82523]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:42:49.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:49 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:42:49.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:49 smithi172 ceph-mon[65307]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:42:49.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:49 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:42:50.066 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:42:50.066 INFO:teuthology.orchestra.run.smithi145.stdout: "target_image": null, 2024-04-22T15:42:50.066 INFO:teuthology.orchestra.run.smithi145.stdout: "in_progress": false, 2024-04-22T15:42:50.066 INFO:teuthology.orchestra.run.smithi145.stdout: "which": "", 2024-04-22T15:42:50.066 INFO:teuthology.orchestra.run.smithi145.stdout: "services_complete": [], 2024-04-22T15:42:50.067 INFO:teuthology.orchestra.run.smithi145.stdout: "progress": null, 2024-04-22T15:42:50.067 INFO:teuthology.orchestra.run.smithi145.stdout: "message": "", 2024-04-22T15:42:50.067 INFO:teuthology.orchestra.run.smithi145.stdout: "is_paused": false 2024-04-22T15:42:50.067 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:42:50.701 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph health detail' 2024-04-22T15:42:51.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:51 smithi145 ceph-mon[82523]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:42:51.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:51 smithi145 ceph-mon[82523]: from='client.44637 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:42:51.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:51 smithi145 ceph-mon[88162]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:42:51.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:51 smithi145 ceph-mon[88162]: from='client.44637 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:42:51.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:51 smithi172 ceph-mon[65307]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:42:51.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:51 smithi172 ceph-mon[65307]: from='client.44637 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:42:52.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:52 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:42:52.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:52 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:42:52.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:52 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:42:52.818 INFO:teuthology.orchestra.run.smithi145.stdout:HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:42:52.818 INFO:teuthology.orchestra.run.smithi145.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:42:52.818 INFO:teuthology.orchestra.run.smithi145.stdout: daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state 2024-04-22T15:42:53.441 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:53 smithi145 ceph-mon[88162]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:42:53.441 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:53 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/905231525' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:42:53.441 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:53 smithi145 ceph-mon[82523]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:42:53.441 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:53 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/905231525' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:42:53.465 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo' 2024-04-22T15:42:53.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:53 smithi172 ceph-mon[65307]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:42:53.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:53 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/905231525' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:42:55.351 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:42:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:42:55] "GET /metrics HTTP/1.1" 200 38387 "" "Prometheus/2.43.0" 2024-04-22T15:42:55.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:55 smithi145 ceph-mon[88162]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:42:55.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:55 smithi145 ceph-mon[82523]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:42:55.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:55 smithi172 ceph-mon[65307]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:42:57.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:57 smithi145 ceph-mon[88162]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:42:57.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:57 smithi145 ceph-mon[88162]: from='client.54586 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:42:57.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:57 smithi145 ceph-mon[82523]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:42:57.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:57 smithi145 ceph-mon[82523]: from='client.54586 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:42:57.699 INFO:teuthology.orchestra.run.smithi145.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:42:57.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:57 smithi172 ceph-mon[65307]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:42:57.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:57 smithi172 ceph-mon[65307]: from='client.54586 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:42:58.635 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-22T15:42:58.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:58 smithi145 ceph-mon[88162]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:42:58.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:58 smithi145 ceph-mon[88162]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:42:58.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:58 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:58.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:58 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:42:58.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:58 smithi145 ceph-mon[82523]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:42:58.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:58 smithi145 ceph-mon[82523]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:42:58.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:58 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:58.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:58 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:42:58.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:58 smithi172 ceph-mon[65307]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:42:58.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:58 smithi172 ceph-mon[65307]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:42:58.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:58 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:58.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:58 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:42:59.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:59 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:42:59.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:59 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:42:59.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:59 smithi172 ceph-mon[65307]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:42:59.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:59 smithi172 ceph-mon[65307]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:42:59.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:59 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:59.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:59 smithi172 ceph-mon[65307]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:42:59.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:59 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:42:59.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:59 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:42:59.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:42:59 smithi172 ceph-mon[65307]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:42:59.990 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:59 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:42:59.990 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:59 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:42:59.991 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:59 smithi145 ceph-mon[88162]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:42:59.991 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:59 smithi145 ceph-mon[88162]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:42:59.991 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:59 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:59.991 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:59 smithi145 ceph-mon[88162]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:42:59.991 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:59 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:42:59.991 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:59 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:42:59.991 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:42:59 smithi145 ceph-mon[88162]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:42:59.991 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:59 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:42:59.991 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:59 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:42:59.991 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:59 smithi145 ceph-mon[82523]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-22T15:42:59.992 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:59 smithi145 ceph-mon[82523]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:42:59.992 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:59 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:42:59.992 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:59 smithi145 ceph-mon[82523]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:42:59.992 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:59 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:42:59.992 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:59 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:42:59.992 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:42:59 smithi145 ceph-mon[82523]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:43:00.674 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:43:01.345 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:43:01.345 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (9m) 2m ago 32m 27.0M - 0.25.0 c8568f914cd2 1c58ccf37ffd 2024-04-22T15:43:01.345 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (9m) 35s ago 31m 79.5M - 9.4.12 2bacad6d85d8 2f825d15ab10 2024-04-22T15:43:01.345 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 unknown 2m ago 31m 60.7M - 3.5 e1d6a67b021e 9704631ffa0e 2024-04-22T15:43:01.345 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443,9283,8765 running (11m) 35s ago 36m 454M - 19.0.0-3223-g5883d494 3d983c26e278 e7500ef71604 2024-04-22T15:43:01.345 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (21m) 2m ago 38m 511M - 19.0.0-3223-g5883d494 3d983c26e278 b2de97566cf4 2024-04-22T15:43:01.345 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (7m) 2m ago 38m 53.7M 2048M 19.0.0-3223-g5883d494 3d983c26e278 7bfdc11120f4 2024-04-22T15:43:01.345 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (8m) 35s ago 37m 51.7M 2048M 19.0.0-3223-g5883d494 3d983c26e278 0a360b08a6d6 2024-04-22T15:43:01.345 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (6m) 2m ago 37m 42.0M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38dcfa677c18 2024-04-22T15:43:01.345 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (10m) 2m ago 32m 15.3M - 1.5.0 0da6a335fe13 8c75c26fec05 2024-04-22T15:43:01.345 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (10m) 35s ago 32m 15.8M - 1.5.0 0da6a335fe13 8fd2949e2eaf 2024-04-22T15:43:01.346 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (5m) 2m ago 36m 51.7M 2992M 19.0.0-3223-g5883d494 3d983c26e278 e4bc06317af0 2024-04-22T15:43:01.346 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (4m) 2m ago 36m 44.0M 2992M 19.0.0-3223-g5883d494 3d983c26e278 d299f622f08d 2024-04-22T15:43:01.346 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (3m) 2m ago 35m 41.7M 2992M 19.0.0-3223-g5883d494 3d983c26e278 6aea4a926fe4 2024-04-22T15:43:01.346 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (2m) 2m ago 35m 15.1M 2992M 19.0.0-3223-g5883d494 3d983c26e278 c0cd2bdc1439 2024-04-22T15:43:01.346 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (2m) 35s ago 34m 47.6M 3248M 19.0.0-3223-g5883d494 3d983c26e278 7ac5776bc07a 2024-04-22T15:43:01.346 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (110s) 35s ago 34m 52.8M 3248M 19.0.0-3223-g5883d494 3d983c26e278 5f803668a17b 2024-04-22T15:43:01.346 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (74s) 35s ago 34m 45.8M 3248M 19.0.0-3223-g5883d494 3d983c26e278 9e049dd4678d 2024-04-22T15:43:01.346 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (39s) 35s ago 33m 12.5M 3248M 19.0.0-3223-g5883d494 3d983c26e278 951a8da56333 2024-04-22T15:43:01.346 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (10m) 35s ago 32m 77.0M - 2.43.0 a07b618ecd1d 51cf4df6e8d4 2024-04-22T15:43:01.346 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (31m) 2m ago 31m 80.2M - 17.2.0 e1d6a67b021e c90a636c947e 2024-04-22T15:43:01.346 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (31m) 35s ago 31m 78.7M - 17.2.0 e1d6a67b021e 20ac46e9bfda 2024-04-22T15:43:01.757 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:43:01.757 INFO:teuthology.orchestra.run.smithi145.stdout: "mon": { 2024-04-22T15:43:01.757 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 3 2024-04-22T15:43:01.757 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:43:01.757 INFO:teuthology.orchestra.run.smithi145.stdout: "mgr": { 2024-04-22T15:43:01.757 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:43:01.757 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:43:01.757 INFO:teuthology.orchestra.run.smithi145.stdout: "osd": { 2024-04-22T15:43:01.758 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 8 2024-04-22T15:43:01.758 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:43:01.758 INFO:teuthology.orchestra.run.smithi145.stdout: "rgw": { 2024-04-22T15:43:01.758 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T15:43:01.758 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:43:01.758 INFO:teuthology.orchestra.run.smithi145.stdout: "overall": { 2024-04-22T15:43:01.758 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-04-22T15:43:01.758 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 13 2024-04-22T15:43:01.758 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:43:01.758 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:43:01.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:43:01.642+0000 7f17b18ae700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:01.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:43:01.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:01.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:01.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:01.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:01.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:01.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:43:01.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:01.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:43:01.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:01.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:43:01.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:01.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:01.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:01.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:01.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:01.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:01.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:01.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:01.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:01.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:43:01.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:43:01.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:43:01.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:43:01.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:43:01.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:43:01.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:43:01.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:43:01.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:43:01.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:43:01.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:43:01.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:43:01.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:43:01.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:43:01.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:43:01.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:43:01.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:43:01.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:01.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:43:01.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:01.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:01.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:01.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:01.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:01.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:43:01.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:01.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:43:01.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:01.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:43:01.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:01.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:01.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:01.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:01.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:01.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:01.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:01.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:01.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:02.118 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:43:02.118 INFO:teuthology.orchestra.run.smithi145.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", 2024-04-22T15:43:02.119 INFO:teuthology.orchestra.run.smithi145.stdout: "in_progress": true, 2024-04-22T15:43:02.119 INFO:teuthology.orchestra.run.smithi145.stdout: "which": "Upgrading daemons in service(s) rgw.foo", 2024-04-22T15:43:02.119 INFO:teuthology.orchestra.run.smithi145.stdout: "services_complete": [], 2024-04-22T15:43:02.119 INFO:teuthology.orchestra.run.smithi145.stdout: "progress": "", 2024-04-22T15:43:02.119 INFO:teuthology.orchestra.run.smithi145.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa image", 2024-04-22T15:43:02.119 INFO:teuthology.orchestra.run.smithi145.stdout: "is_paused": false 2024-04-22T15:43:02.119 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:43:02.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:01 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:43:02.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:01 smithi172 ceph-mon[65307]: from='client.54592 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:02.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:01 smithi172 ceph-mon[65307]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:43:02.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:01 smithi172 ceph-mon[65307]: from='client.54598 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:02.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:01 smithi172 ceph-mon[65307]: from='client.44667 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:02.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:01 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/1559601899' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:02.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:43:02.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-mon[88162]: from='client.54592 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:02.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-mon[88162]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:43:02.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-mon[88162]: from='client.54598 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:02.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-mon[88162]: from='client.44667 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:02.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/1559601899' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:02.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:43:02.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-mon[82523]: from='client.54592 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:02.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-mon[82523]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:43:02.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-mon[82523]: from='client.54598 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:02.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-mon[82523]: from='client.44667 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:02.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:01 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/1559601899' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:03.085 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:02 smithi145 ceph-mon[88162]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:03.086 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:43:03.086 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:03.086 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:03.086 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:03.086 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:03.086 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:03.086 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:43:03.086 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:03.086 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:43:03.086 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:03.086 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:43:03.086 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:03.087 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:03.087 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:03.087 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:03.087 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:03.087 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:03.087 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:03.087 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:03.087 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:03.087 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:43:03.087 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:43:03.087 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:43:03.087 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:43:03.087 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:43:03.087 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:43:03.088 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:43:03.088 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:43:03.088 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:43:03.088 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:43:03.088 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:43:03.088 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:43:03.088 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:43:03.088 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:43:03.088 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:43:03.088 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:43:03.088 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:43:03.088 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:03.088 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:43:03.088 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:03.088 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:03.088 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:03.089 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:03.089 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:03.089 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:43:03.089 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:03.089 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:43:03.089 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:03.089 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:43:03.089 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:03.089 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:03.089 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:03.089 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:03.090 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:02 smithi145 ceph-mon[82523]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:03.090 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:43:03.090 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:03.090 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:03.090 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:03.090 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:03.090 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:03.090 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:43:03.090 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:03.090 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:43:03.090 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:03.090 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:43:03.090 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:03.090 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:03.091 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:03.091 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:03.091 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:03.091 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:03.091 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:03.091 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:03.091 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:03.091 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:43:03.091 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:43:03.091 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:43:03.091 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:43:03.091 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:43:03.091 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:43:03.091 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:43:03.092 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:43:03.092 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:43:03.092 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:43:03.092 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:43:03.092 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:43:03.092 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:43:03.092 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:43:03.092 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:43:03.092 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:43:03.092 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:43:03.092 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:03.092 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:43:03.092 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:03.092 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:03.092 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:03.093 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:03.093 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:03.093 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:43:03.093 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:03.093 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:43:03.093 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:03.093 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:43:03.093 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:03.093 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:03.093 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:03.093 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:03.093 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:03.094 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:03.094 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:03.094 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:03.094 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:03.094 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:02 smithi145 ceph-mon[88162]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:43:03.094 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:02 smithi145 ceph-mon[88162]: from='client.54616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:03.094 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:03.094 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:03.094 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:03.094 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:03.094 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:03.094 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:02 smithi145 ceph-mon[82523]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:43:03.095 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:02 smithi145 ceph-mon[82523]: from='client.54616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:03.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:02 smithi172 ceph-mon[65307]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:03.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:43:03.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:03.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:03.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:03.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:03.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:03.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:43:03.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:03.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:43:03.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:03.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:43:03.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:03.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:03.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:03.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:03.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:03.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:03.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:03.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:03.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:03.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:43:03.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:43:03.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:43:03.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:43:03.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:43:03.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:43:03.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:43:03.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:43:03.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:43:03.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:43:03.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:43:03.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:43:03.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:43:03.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:43:03.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:43:03.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:43:03.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:43:03.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:03.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:43:03.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:03.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:03.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:03.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:03.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:03.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:43:03.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:03.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:43:03.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:03.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:43:03.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:03.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:03.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:03.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:03.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:03.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:03.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:03.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:03.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:03.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:02 smithi172 ceph-mon[65307]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:43:03.223 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:02 smithi172 ceph-mon[65307]: from='client.54616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:04.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:03 smithi172 ceph-mon[65307]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 547 B/s rd, 0 op/s 2024-04-22T15:43:04.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:03 smithi145 ceph-mon[82523]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 547 B/s rd, 0 op/s 2024-04-22T15:43:04.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:03 smithi145 ceph-mon[88162]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 547 B/s rd, 0 op/s 2024-04-22T15:43:05.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:05.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[82523]: Upgrade: Target is version 19.0.0-3223-g5883d494 (squid) 2024-04-22T15:43:05.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[82523]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, digests ['quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb'] 2024-04-22T15:43:05.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:43:05.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:05.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all mgr 2024-04-22T15:43:05.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:05.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:05.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all mon 2024-04-22T15:43:05.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:05.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:05.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all crash 2024-04-22T15:43:05.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:05.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:05.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all osd 2024-04-22T15:43:05.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:05.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:05.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all mds 2024-04-22T15:43:05.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:05.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:43:05.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:05.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[88162]: Upgrade: Target is version 19.0.0-3223-g5883d494 (squid) 2024-04-22T15:43:05.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[88162]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, digests ['quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb'] 2024-04-22T15:43:05.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:43:05.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:05.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all mgr 2024-04-22T15:43:05.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:05.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:05.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all mon 2024-04-22T15:43:05.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:05.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:05.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all crash 2024-04-22T15:43:05.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:05.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:05.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all osd 2024-04-22T15:43:05.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:05.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:05.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all mds 2024-04-22T15:43:05.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:05.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:04 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:43:05.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:04 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:04 smithi172 ceph-mon[65307]: Upgrade: Target is version 19.0.0-3223-g5883d494 (squid) 2024-04-22T15:43:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:04 smithi172 ceph-mon[65307]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, digests ['quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb'] 2024-04-22T15:43:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:04 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:43:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:04 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:04 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all mgr 2024-04-22T15:43:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:04 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:04 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:04 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all mon 2024-04-22T15:43:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:04 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:04 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:04 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all crash 2024-04-22T15:43:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:04 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:05.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:04 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:05.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:04 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all osd 2024-04-22T15:43:05.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:04 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:05.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:04 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:05.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:04 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all mds 2024-04-22T15:43:05.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:04 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:05.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:04 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:43:05.583 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:05 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:43:05] "GET /metrics HTTP/1.1" 200 38389 "" "Prometheus/2.43.0" 2024-04-22T15:43:06.294 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:06 smithi145 ceph-mon[88162]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:43:06.295 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:06 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:06.295 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:06 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi145.ygrqmi", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T15:43:06.295 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:06 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:06.295 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:06 smithi145 ceph-mon[82523]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:43:06.295 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:06 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:06.295 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:06 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi145.ygrqmi", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T15:43:06.295 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:06 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:06.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:06 smithi172 ceph-mon[65307]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:43:06.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:06 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:06.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:06 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi145.ygrqmi", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T15:43:06.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:06 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:07.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:07 smithi145 ceph-mon[88162]: Upgrade: Updating rgw.foo.smithi145.ygrqmi (1/2) 2024-04-22T15:43:07.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:07 smithi145 ceph-mon[88162]: Deploying daemon rgw.foo.smithi145.ygrqmi on smithi145 2024-04-22T15:43:07.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:07 smithi145 ceph-mon[82523]: Upgrade: Updating rgw.foo.smithi145.ygrqmi (1/2) 2024-04-22T15:43:07.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:07 smithi145 ceph-mon[82523]: Deploying daemon rgw.foo.smithi145.ygrqmi on smithi145 2024-04-22T15:43:07.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:07 smithi172 ceph-mon[65307]: Upgrade: Updating rgw.foo.smithi145.ygrqmi (1/2) 2024-04-22T15:43:07.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:07 smithi172 ceph-mon[65307]: Deploying daemon rgw.foo.smithi145.ygrqmi on smithi145 2024-04-22T15:43:08.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:08 smithi145 ceph-mon[88162]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:43:08.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:08 smithi145 ceph-mon[82523]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:43:08.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:08 smithi172 ceph-mon[65307]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:43:10.270 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:10 smithi172 ceph-mon[65307]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:43:10.270 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:10 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:10.271 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:10 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:10.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:10 smithi145 ceph-mon[82523]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:43:10.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:10 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:10.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:10 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:10.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:10 smithi145 ceph-mon[88162]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:43:10.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:10 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:10.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:10 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:12.682 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:12 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:43:12.682 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:12 smithi172 ceph-mon[65307]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 23 op/s 2024-04-22T15:43:12.682 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:12 smithi172 ceph-mon[65307]: Upgrade: Updating rgw.foo.smithi172.glxwwa (2/2) 2024-04-22T15:43:12.683 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:12 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:12.683 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:12 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.glxwwa", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T15:43:12.683 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:12 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:12.683 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:12 smithi172 ceph-mon[65307]: Deploying daemon rgw.foo.smithi172.glxwwa on smithi172 2024-04-22T15:43:12.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:12 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:43:12.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:12 smithi145 ceph-mon[88162]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 23 op/s 2024-04-22T15:43:12.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:12 smithi145 ceph-mon[88162]: Upgrade: Updating rgw.foo.smithi172.glxwwa (2/2) 2024-04-22T15:43:12.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:12 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:12.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:12 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.glxwwa", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T15:43:12.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:12 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:12.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:12 smithi145 ceph-mon[88162]: Deploying daemon rgw.foo.smithi172.glxwwa on smithi172 2024-04-22T15:43:12.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:12 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:43:12.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:12 smithi145 ceph-mon[82523]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 23 op/s 2024-04-22T15:43:12.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:12 smithi145 ceph-mon[82523]: Upgrade: Updating rgw.foo.smithi172.glxwwa (2/2) 2024-04-22T15:43:12.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:12 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:12.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:12 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi172.glxwwa", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T15:43:12.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:12 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:12.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:12 smithi145 ceph-mon[82523]: Deploying daemon rgw.foo.smithi172.glxwwa on smithi172 2024-04-22T15:43:14.652 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:14 smithi172 ceph-mon[65307]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 56 op/s 2024-04-22T15:43:14.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:14 smithi145 ceph-mon[82523]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 56 op/s 2024-04-22T15:43:14.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:14 smithi145 ceph-mon[88162]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 56 op/s 2024-04-22T15:43:15.622 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:43:15] "GET /metrics HTTP/1.1" 200 38389 "" "Prometheus/2.43.0" 2024-04-22T15:43:16.173 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:16 smithi172 ceph-mon[65307]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 85 B/s wr, 84 op/s 2024-04-22T15:43:16.173 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:16 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:16.173 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:16 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:16.173 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:16 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:43:16.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:16 smithi145 ceph-mon[88162]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 85 B/s wr, 84 op/s 2024-04-22T15:43:16.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:16 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:16.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:16 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:16.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:16 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:43:16.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:16 smithi145 ceph-mon[82523]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 85 B/s wr, 84 op/s 2024-04-22T15:43:16.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:16 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:16.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:16 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:16.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:16 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:43:18.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:18 smithi145 ceph-mon[82523]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s rd, 85 B/s wr, 118 op/s 2024-04-22T15:43:18.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:18 smithi145 ceph-mon[88162]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s rd, 85 B/s wr, 118 op/s 2024-04-22T15:43:18.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:18 smithi172 ceph-mon[65307]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s rd, 85 B/s wr, 118 op/s 2024-04-22T15:43:19.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:19 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:19.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:19 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:19.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:19 smithi145 ceph-mon[82523]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 102 KiB/s rd, 170 B/s wr, 158 op/s 2024-04-22T15:43:19.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:19 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:19.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:19 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:19.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:19 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:43:19.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:19 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:19.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:19 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:19.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:19 smithi145 ceph-mon[88162]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 102 KiB/s rd, 170 B/s wr, 158 op/s 2024-04-22T15:43:19.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:19 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:19.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:19 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:19.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:19 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:43:19.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:19 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:19.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:19 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:19.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:19 smithi172 ceph-mon[65307]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 102 KiB/s rd, 170 B/s wr, 158 op/s 2024-04-22T15:43:19.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:19 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:19.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:19 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:19.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:19 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:43:20.773 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:20 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:20.773 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:20 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:20.774 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:20 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:20.774 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:20 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:20.774 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:20 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:20.774 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:20 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:20.774 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:20 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:20.774 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:20 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:20.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:20 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:20.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:20 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:20.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:20 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:20.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:20 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:21 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:43:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:21 smithi172 ceph-mon[65307]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 167 op/s 2024-04-22T15:43:21.785 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:21 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:43:21.785 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:21 smithi145 ceph-mon[88162]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 167 op/s 2024-04-22T15:43:21.785 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:21 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:43:21.785 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:21 smithi145 ceph-mon[82523]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 167 op/s 2024-04-22T15:43:24.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:23 smithi172 ceph-mon[65307]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 94 KiB/s rd, 170 B/s wr, 144 op/s 2024-04-22T15:43:24.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:23 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:24.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:23 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:24.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:23 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:24.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:23 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:24.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:23 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:24.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:23 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:43:24.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:23 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:24.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:23 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:43:24.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:23 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:24.353 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:23 smithi145 ceph-mon[82523]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 94 KiB/s rd, 170 B/s wr, 144 op/s 2024-04-22T15:43:24.353 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:23 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:24.353 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:23 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:24.353 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:23 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:24.353 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:23 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:24.353 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:23 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:24.353 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:23 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:43:24.353 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:23 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:24.353 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:23 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:43:24.353 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:23 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:24.354 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:23 smithi145 ceph-mon[88162]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 94 KiB/s rd, 170 B/s wr, 144 op/s 2024-04-22T15:43:24.354 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:23 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:24.354 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:23 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:24.354 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:23 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:24.354 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:23 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:24.354 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:23 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:24.354 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:23 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:43:24.354 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:23 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:24.354 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:23 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:43:24.354 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:23 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:25.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:24 smithi172 ceph-mon[65307]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 82 KiB/s rd, 190 B/s wr, 125 op/s 2024-04-22T15:43:25.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:24 smithi172 ceph-mon[65307]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 75 KiB/s rd, 116 B/s wr, 115 op/s 2024-04-22T15:43:25.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:24 smithi172 ceph-mon[65307]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:43:25.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:24 smithi172 ceph-mon[65307]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:43:25.290 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:24 smithi145 ceph-mon[82523]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 82 KiB/s rd, 190 B/s wr, 125 op/s 2024-04-22T15:43:25.290 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:24 smithi145 ceph-mon[82523]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 75 KiB/s rd, 116 B/s wr, 115 op/s 2024-04-22T15:43:25.290 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:24 smithi145 ceph-mon[82523]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:43:25.290 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:24 smithi145 ceph-mon[82523]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:43:25.290 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:24 smithi145 ceph-mon[88162]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 82 KiB/s rd, 190 B/s wr, 125 op/s 2024-04-22T15:43:25.290 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:24 smithi145 ceph-mon[88162]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 75 KiB/s rd, 116 B/s wr, 115 op/s 2024-04-22T15:43:25.290 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:24 smithi145 ceph-mon[88162]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:43:25.290 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:24 smithi145 ceph-mon[88162]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:43:25.622 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:43:25] "GET /metrics HTTP/1.1" 200 38419 "" "Prometheus/2.43.0" 2024-04-22T15:43:25.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:43:25.466+0000 7f17b18ae700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:25.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:43:25.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:25.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:25.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:25.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:25.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:25.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:43:25.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:25.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:43:25.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:25.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:43:25.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:25.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:25.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:25.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:25.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:25.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:25.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:25.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:25.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:25.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:43:25.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:43:25.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:43:25.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:43:25.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:43:25.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:43:25.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:43:25.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:43:25.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:43:25.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:43:25.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:43:25.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:43:25.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:43:25.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:43:25.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:43:25.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:43:25.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:43:25.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:25.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:43:25.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:25.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:25.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:25.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:25.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:25.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:43:25.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:25.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:43:25.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:25.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:43:25.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:25.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:25.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:25.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:25.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:25.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:25.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:25.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:25.627 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:26.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:43:26.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:26.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi145.ygrqmi"}]: dispatch 2024-04-22T15:43:26.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi145.ygrqmi"}]': finished 2024-04-22T15:43:26.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi172.glxwwa"}]: dispatch 2024-04-22T15:43:26.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi172.glxwwa"}]': finished 2024-04-22T15:43:26.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:26.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:26.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:26.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:26.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.219 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T15:43:26.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T15:43:26.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:43:26.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T15:43:26.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T15:43:26.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T15:43:26.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T15:43:26.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-22T15:43:26.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T15:43:26.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T15:43:26.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T15:43:26.220 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-04-22T15:43:26.221 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T15:43:26.221 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T15:43:26.221 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:43:26.221 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T15:43:26.221 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T15:43:26.221 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T15:43:26.221 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T15:43:26.221 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T15:43:26.221 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T15:43:26.221 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T15:43:26.221 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:43:26.221 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:43:26.221 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:43:26.221 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:43:26.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:43:26.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:43:26.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T15:43:26.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T15:43:26.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:43:26.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:26.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:43:26.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:25 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:26.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:43:26.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:26.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi145.ygrqmi"}]: dispatch 2024-04-22T15:43:26.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi145.ygrqmi"}]': finished 2024-04-22T15:43:26.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi172.glxwwa"}]: dispatch 2024-04-22T15:43:26.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi172.glxwwa"}]': finished 2024-04-22T15:43:26.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:26.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:26.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:26.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:26.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:43:26.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:26.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi145.ygrqmi"}]: dispatch 2024-04-22T15:43:26.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi145.ygrqmi"}]': finished 2024-04-22T15:43:26.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi172.glxwwa"}]: dispatch 2024-04-22T15:43:26.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi172.glxwwa"}]': finished 2024-04-22T15:43:26.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:26.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:26.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:26.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:26.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.376 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:26.376 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T15:43:26.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T15:43:26.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:43:26.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T15:43:26.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T15:43:26.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T15:43:26.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T15:43:26.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-22T15:43:26.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T15:43:26.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T15:43:26.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T15:43:26.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-04-22T15:43:26.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T15:43:26.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T15:43:26.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:43:26.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T15:43:26.378 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T15:43:26.378 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T15:43:26.378 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T15:43:26.378 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T15:43:26.378 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T15:43:26.378 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T15:43:26.378 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:43:26.378 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:43:26.378 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:43:26.378 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:43:26.378 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:43:26.378 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:43:26.378 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T15:43:26.378 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T15:43:26.379 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:43:26.379 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:26.379 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:43:26.379 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:26.382 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-22T15:43:26.382 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-22T15:43:26.382 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:43:26.383 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-22T15:43:26.383 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-22T15:43:26.383 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-22T15:43:26.383 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-22T15:43:26.383 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-22T15:43:26.383 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-22T15:43:26.383 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-22T15:43:26.383 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-22T15:43:26.383 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-04-22T15:43:26.383 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-22T15:43:26.383 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-22T15:43:26.383 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:43:26.383 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-22T15:43:26.383 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-22T15:43:26.384 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-22T15:43:26.384 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-22T15:43:26.384 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-22T15:43:26.384 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-22T15:43:26.384 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-22T15:43:26.384 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:43:26.384 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:43:26.384 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:43:26.384 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:43:26.384 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:43:26.385 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-22T15:43:26.385 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-22T15:43:26.385 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-22T15:43:26.385 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:43:26.385 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:26.385 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:43:26.385 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:25 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:27.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:26 smithi145 ceph-mon[82523]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:27.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:43:27.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:27.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:27.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:27.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:27.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:27.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:43:27.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:27.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:43:27.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:27.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:43:27.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:27.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:27.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:27.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:27.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:27.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:27.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:27.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:27.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:27.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:43:27.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:43:27.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:43:27.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:43:27.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:43:27.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:43:27.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:43:27.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:43:27.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:43:27.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:43:27.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:43:27.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:43:27.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:43:27.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:43:27.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:43:27.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:43:27.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:43:27.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:27.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:43:27.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:27.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:27.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:27.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:27.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:27.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:43:27.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:27.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:43:27.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:27.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:43:27.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:27.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:27.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:27.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:27.127 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:26 smithi145 ceph-mon[88162]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:27.127 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:43:27.127 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:27.127 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:27.127 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:27.127 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:27.127 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:27.127 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:43:27.127 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:27.127 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:43:27.127 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:27.127 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:43:27.127 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:27.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:27.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:27.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:27.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:27.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:27.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:27.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:27.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:27.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:43:27.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:43:27.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:43:27.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:43:27.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:43:27.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:43:27.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:43:27.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:43:27.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:43:27.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:43:27.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:43:27.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:43:27.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:43:27.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:43:27.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:43:27.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:43:27.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:43:27.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:27.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:43:27.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:27.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:27.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:27.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:27.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:27.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:43:27.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:27.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:43:27.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:27.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:43:27.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:27.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:27.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:27.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:27.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:27.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:27.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:27.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:27.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:27.131 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:26 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all rgw 2024-04-22T15:43:27.131 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:26 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T15:43:27.131 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:26 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T15:43:27.131 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:26 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all nfs 2024-04-22T15:43:27.131 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:26 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all nvmeof 2024-04-22T15:43:27.131 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:26 smithi145 ceph-mon[88162]: Upgrade: Finalizing container_image settings 2024-04-22T15:43:27.131 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:26 smithi145 ceph-mon[88162]: Upgrade: Complete! 2024-04-22T15:43:27.131 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:26 smithi145 ceph-mon[88162]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s rd, 116 B/s wr, 68 op/s 2024-04-22T15:43:27.131 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:26 smithi145 ceph-mon[88162]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 0 B/s wr, 18 op/s 2024-04-22T15:43:27.132 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:26 smithi145 ceph-mon[88162]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:43:27.132 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:26 smithi145 ceph-mon[88162]: Checking dashboard <-> RGW credentials 2024-04-22T15:43:27.132 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:26 smithi145 ceph-mon[88162]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:43:27.132 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:26 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:43:27.132 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:26 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:27.132 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:26 smithi145 ceph-mon[88162]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:43:27.132 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:27.132 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:27.132 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:27.132 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:27.132 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:27.132 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:26 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all rgw 2024-04-22T15:43:27.132 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:26 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T15:43:27.133 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:26 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T15:43:27.133 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:26 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all nfs 2024-04-22T15:43:27.133 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:26 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all nvmeof 2024-04-22T15:43:27.133 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:26 smithi145 ceph-mon[82523]: Upgrade: Finalizing container_image settings 2024-04-22T15:43:27.133 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:26 smithi145 ceph-mon[82523]: Upgrade: Complete! 2024-04-22T15:43:27.133 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:26 smithi145 ceph-mon[82523]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s rd, 116 B/s wr, 68 op/s 2024-04-22T15:43:27.133 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:26 smithi145 ceph-mon[82523]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 0 B/s wr, 18 op/s 2024-04-22T15:43:27.133 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:26 smithi145 ceph-mon[82523]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:43:27.133 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:26 smithi145 ceph-mon[82523]: Checking dashboard <-> RGW credentials 2024-04-22T15:43:27.133 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:26 smithi145 ceph-mon[82523]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:43:27.133 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:26 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:43:27.133 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:26 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:27.133 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:26 smithi145 ceph-mon[82523]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:43:27.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:26 smithi172 ceph-mon[65307]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:27.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:43:27.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:27.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:27.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:27.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:27.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:27.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:43:27.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:27.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:43:27.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:27.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:43:27.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:27.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:27.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:27.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:27.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:27.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:27.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:27.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:27.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:27.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:43:27.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:43:27.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:43:27.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:43:27.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:43:27.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:43:27.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:43:27.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:43:27.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:43:27.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:43:27.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:43:27.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:43:27.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:43:27.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:43:27.223 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:43:27.223 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:43:27.223 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:43:27.223 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:27.223 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:43:27.223 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:27.223 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:27.223 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:27.223 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:27.223 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:27.223 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:43:27.223 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:27.223 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:43:27.223 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:27.224 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:43:27.224 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:27.224 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:27.224 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:27.224 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:27.224 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:27.224 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:27.224 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:27.224 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:27.224 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:27.224 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:26 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all rgw 2024-04-22T15:43:27.224 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:26 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T15:43:27.225 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:26 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T15:43:27.225 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:26 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all nfs 2024-04-22T15:43:27.225 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:26 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all nvmeof 2024-04-22T15:43:27.225 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:26 smithi172 ceph-mon[65307]: Upgrade: Finalizing container_image settings 2024-04-22T15:43:27.225 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:26 smithi172 ceph-mon[65307]: Upgrade: Complete! 2024-04-22T15:43:27.225 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:26 smithi172 ceph-mon[65307]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s rd, 116 B/s wr, 68 op/s 2024-04-22T15:43:27.225 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:26 smithi172 ceph-mon[65307]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 0 B/s wr, 18 op/s 2024-04-22T15:43:27.225 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:26 smithi172 ceph-mon[65307]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:43:27.225 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:26 smithi172 ceph-mon[65307]: Checking dashboard <-> RGW credentials 2024-04-22T15:43:27.225 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:26 smithi172 ceph-mon[65307]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:43:27.225 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:26 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:43:27.225 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:26 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:27.225 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:26 smithi172 ceph-mon[65307]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:43:28.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:43:27.902+0000 7f17b18ae700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:28.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:43:28.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:28.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:28.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:28.123 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:28.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:28.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:43:28.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:28.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:43:28.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:28.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:43:28.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:28.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:28.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:28.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:28.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:28.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:28.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:28.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:28.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:28.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:43:28.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:43:28.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:43:28.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:43:28.125 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:43:28.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:43:28.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:43:28.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:43:28.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:43:28.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:43:28.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:43:28.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:43:28.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:43:28.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:43:28.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:43:28.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:43:28.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:43:28.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:28.126 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:43:28.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:28.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:28.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:28.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:28.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:28.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:43:28.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:28.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:43:28.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:28.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:43:28.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:28.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:28.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:28.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:28.127 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:28.128 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:28.128 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:28.128 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:28.128 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:27 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:28.912 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:28 smithi145 ceph-mon[82523]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 2 op/s 2024-04-22T15:43:28.912 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:28 smithi145 ceph-mon[82523]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:28.912 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:43:28.912 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:28.912 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:28.912 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:28.912 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:28.912 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:28.912 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:43:28.912 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:28.912 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:43:28.912 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:28.913 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:43:28.913 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:28.913 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:28.913 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:28.913 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:28.913 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:28.913 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:28.913 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:28.913 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:28.913 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:28.913 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:43:28.913 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:43:28.913 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:43:28.913 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:43:28.913 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:43:28.914 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:43:28.914 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:43:28.914 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:43:28.914 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:43:28.914 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:43:28.914 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:43:28.914 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:43:28.914 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:43:28.914 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:43:28.914 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:43:28.914 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:43:28.914 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:43:28.914 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:28.914 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:43:28.915 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:28.915 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:28.915 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:28.915 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:28.915 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:28.915 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:43:28.915 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:28.915 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:43:28.915 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:28.915 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:43:28.915 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:28.915 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:28.915 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:28.915 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:28.916 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:28.916 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:28.916 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:28.916 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:28.916 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:28.916 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:28 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:43:28.916 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:28 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:28.916 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:28 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:43:28.916 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:28 smithi145 ceph-mon[82523]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 952 B/s rd, 1 op/s 2024-04-22T15:43:28.916 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:28 smithi145 ceph-mon[82523]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 952 B/s rd, 1 op/s 2024-04-22T15:43:28.916 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:28 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:28.916 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:28 smithi145 ceph-mon[82523]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:43:28.916 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:28 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:43:28.916 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:28 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:28.916 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:28 smithi145 ceph-mon[82523]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:43:29.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:28 smithi172 ceph-mon[65307]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 2 op/s 2024-04-22T15:43:29.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:28 smithi172 ceph-mon[65307]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:29.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:43:29.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:29.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:29.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:29.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:29.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:29.218 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:43:29.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:29.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:43:29.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:29.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:43:29.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:29.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:29.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:29.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:29.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:29.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:29.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:29.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:29.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:29.219 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:43:29.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:43:29.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:43:29.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:43:29.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:43:29.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:43:29.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:43:29.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:43:29.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:43:29.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:43:29.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:43:29.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:43:29.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:43:29.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:43:29.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:43:29.220 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:43:29.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:43:29.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:29.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:43:29.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:29.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:29.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:29.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:29.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:29.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:43:29.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:29.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:43:29.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:29.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:43:29.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:29.221 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:29.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:29.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:29.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:29.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:29.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:29.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:29.222 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:29.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:28 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:43:29.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:28 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:29.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:28 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:43:29.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:28 smithi172 ceph-mon[65307]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 952 B/s rd, 1 op/s 2024-04-22T15:43:29.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:28 smithi172 ceph-mon[65307]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 952 B/s rd, 1 op/s 2024-04-22T15:43:29.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:28 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:29.222 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:28 smithi172 ceph-mon[65307]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:43:29.223 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:28 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:43:29.223 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:28 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:29.223 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:28 smithi172 ceph-mon[65307]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:43:29.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:28 smithi145 ceph-mon[88162]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 2 op/s 2024-04-22T15:43:29.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:28 smithi145 ceph-mon[88162]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:29.373 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:43:29.373 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:29.373 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:29.373 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:29.373 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:29.373 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:29.373 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:43:29.373 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:29.373 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:43:29.373 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:29.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:43:29.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:29.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:29.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:29.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:29.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:29.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:29.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:29.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:29.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:29.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:43:29.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:43:29.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:43:29.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:43:29.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:43:29.375 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:43:29.375 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:43:29.375 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:43:29.375 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:43:29.375 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:43:29.375 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:43:29.375 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:43:29.375 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:43:29.375 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:43:29.375 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:43:29.375 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:43:29.375 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:43:29.375 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:29.375 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:43:29.375 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:29.375 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:29.376 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:29.376 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:29.376 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:29.376 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:43:29.376 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:29.376 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:43:29.376 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:29.376 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:43:29.376 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:29.376 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:29.376 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:29.376 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:29.376 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:29.376 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:29.376 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:29.376 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:29.377 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:29.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:28 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:43:29.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:28 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:29.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:28 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:43:29.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:28 smithi145 ceph-mon[88162]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 952 B/s rd, 1 op/s 2024-04-22T15:43:29.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:28 smithi145 ceph-mon[88162]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 952 B/s rd, 1 op/s 2024-04-22T15:43:29.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:28 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:29.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:28 smithi145 ceph-mon[88162]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:43:29.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:28 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:43:29.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:28 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:29.377 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:28 smithi145 ceph-mon[88162]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:43:30.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:43:29.967+0000 7f17b18ae700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:30.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:43:30.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:30.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:30.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:30.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:30.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:30.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:43:30.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:30.373 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:43:30.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:30.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:43:30.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:30.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:30.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:30.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:30.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:30.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:30.374 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:30.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:30.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:30.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:43:30.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:43:30.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:43:30.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:43:30.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:43:30.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:43:30.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:43:30.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:43:30.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:43:30.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:43:30.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:43:30.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:43:30.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:43:30.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:43:30.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:43:30.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:43:30.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:43:30.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:30.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:43:30.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:30.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:30.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:30.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:30.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:30.376 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:43:30.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:30.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:43:30.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:30.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:43:30.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:30.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:30.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:30.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:30.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:30.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:30.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:30.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:30.377 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:29 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:30.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:30 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:30.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:30 smithi172 ceph-mon[65307]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 5.3 KiB/s rd, 5 op/s 2024-04-22T15:43:30.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:30 smithi172 ceph-mon[65307]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:30.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:43:30.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:30.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:30.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:30.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:30.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:30.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:43:30.968 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:30.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:43:30.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:30.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:43:30.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:30.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:30.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:30.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:30.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:30.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:30.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:30.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:30.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:30.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:43:30.969 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:43:30.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:43:30.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:43:30.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:43:30.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:43:30.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:43:30.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:43:30.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:43:30.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:43:30.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:43:30.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:43:30.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:43:30.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:43:30.970 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:43:30.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:43:30.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:43:30.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:30.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:43:30.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:30.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:30.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:30.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:30.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:30.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:43:30.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:30.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:43:30.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:30.971 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:43:30.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:30.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:30.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:30.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:30.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:30.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:30.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:30.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:30.972 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:31.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:30 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:31.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:30 smithi145 ceph-mon[82523]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 5.3 KiB/s rd, 5 op/s 2024-04-22T15:43:31.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:30 smithi145 ceph-mon[82523]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:31.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:43:31.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:31.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:31.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:31.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:31.123 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:31.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:43:31.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:31.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:43:31.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:31.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:43:31.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:31.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:31.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:31.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:31.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:31.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:31.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:31.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:31.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:31.124 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:43:31.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:43:31.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:43:31.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:43:31.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:43:31.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:43:31.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:43:31.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:43:31.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:43:31.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:43:31.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:43:31.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:43:31.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:43:31.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:43:31.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:43:31.125 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:43:31.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:43:31.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:31.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:43:31.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:31.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:31.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:31.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:31.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:31.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:43:31.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:31.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:43:31.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:31.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:43:31.126 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:31.127 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:31.127 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:31.127 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:31.127 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:31.127 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:31.127 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:31.127 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:31.127 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:31.127 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:30 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:31.127 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:30 smithi145 ceph-mon[88162]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 5.3 KiB/s rd, 5 op/s 2024-04-22T15:43:31.127 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:30 smithi145 ceph-mon[88162]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:31.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:43:31.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:31.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:31.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:31.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:31.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:31.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:43:31.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:31.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:43:31.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:31.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:43:31.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:31.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:31.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:31.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:31.128 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:31.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:31.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:31.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:31.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:31.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:43:31.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:43:31.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:43:31.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:43:31.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:43:31.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:43:31.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:43:31.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:43:31.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:43:31.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:43:31.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:43:31.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:43:31.129 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:43:31.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:43:31.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:43:31.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:43:31.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:43:31.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:31.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:43:31.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:31.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:31.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:31.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:31.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:31.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:43:31.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:31.130 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:43:31.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:31.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:43:31.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:31.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:31.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:31.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:31.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:31.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:31.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:31.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:31.131 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:31.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:31 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:43:32.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:31 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:43:32.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:31 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:43:32.862 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:32 smithi145 ceph-mon[88162]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 4.3 KiB/s rd, 4 op/s 2024-04-22T15:43:32.863 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:32 smithi145 ceph-mon[82523]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 4.3 KiB/s rd, 4 op/s 2024-04-22T15:43:33.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:32 smithi172 ceph-mon[65307]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 4.3 KiB/s rd, 4 op/s 2024-04-22T15:43:33.278 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch ps' 2024-04-22T15:43:33.857 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:33 smithi145 ceph-mon[82523]: from='client.54691 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:33.857 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:33 smithi145 ceph-mon[88162]: from='client.54691 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:34.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:33 smithi172 ceph-mon[65307]: from='client.54691 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:34.218 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:43:33 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=cleanup t=2024-04-22T15:43:33.819089899Z level=info msg="Completed cleanup jobs" duration=2.444247ms 2024-04-22T15:43:35.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:34 smithi145 ceph-mon[88162]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 4 op/s 2024-04-22T15:43:35.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:43:35.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:34 smithi145 ceph-mon[82523]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 4 op/s 2024-04-22T15:43:35.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:43:35.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:34 smithi172 ceph-mon[65307]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 4 op/s 2024-04-22T15:43:35.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:43:35.372 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:43:35] "GET /metrics HTTP/1.1" 200 38422 "" "Prometheus/2.43.0" 2024-04-22T15:43:35.514 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:43:35.514 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (10m) 16s ago 32m 27.0M - 0.25.0 c8568f914cd2 1c58ccf37ffd 2024-04-22T15:43:35.514 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (10m) 17s ago 32m 76.5M - 9.4.12 2bacad6d85d8 2f825d15ab10 2024-04-22T15:43:35.514 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 unknown 16s ago 31m 60.9M - 3.5 e1d6a67b021e 9704631ffa0e 2024-04-22T15:43:35.514 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443,9283,8765 running (11m) 17s ago 37m 454M - 19.0.0-3223-g5883d494 3d983c26e278 e7500ef71604 2024-04-22T15:43:35.515 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (21m) 16s ago 38m 516M - 19.0.0-3223-g5883d494 3d983c26e278 b2de97566cf4 2024-04-22T15:43:35.515 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (7m) 16s ago 39m 62.2M 2048M 19.0.0-3223-g5883d494 3d983c26e278 7bfdc11120f4 2024-04-22T15:43:35.515 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (8m) 17s ago 37m 54.7M 2048M 19.0.0-3223-g5883d494 3d983c26e278 0a360b08a6d6 2024-04-22T15:43:35.515 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (7m) 16s ago 37m 48.4M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38dcfa677c18 2024-04-22T15:43:35.515 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (11m) 16s ago 33m 15.5M - 1.5.0 0da6a335fe13 8c75c26fec05 2024-04-22T15:43:35.515 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (11m) 17s ago 33m 16.0M - 1.5.0 0da6a335fe13 8fd2949e2eaf 2024-04-22T15:43:35.515 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (5m) 16s ago 37m 60.3M 2992M 19.0.0-3223-g5883d494 3d983c26e278 e4bc06317af0 2024-04-22T15:43:35.515 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (5m) 16s ago 36m 52.3M 2992M 19.0.0-3223-g5883d494 3d983c26e278 d299f622f08d 2024-04-22T15:43:35.515 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (4m) 16s ago 36m 48.5M 2992M 19.0.0-3223-g5883d494 3d983c26e278 6aea4a926fe4 2024-04-22T15:43:35.515 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (3m) 16s ago 35m 55.1M 2992M 19.0.0-3223-g5883d494 3d983c26e278 c0cd2bdc1439 2024-04-22T15:43:35.515 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (3m) 17s ago 35m 51.4M 3248M 19.0.0-3223-g5883d494 3d983c26e278 7ac5776bc07a 2024-04-22T15:43:35.515 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (2m) 17s ago 35m 57.0M 3248M 19.0.0-3223-g5883d494 3d983c26e278 5f803668a17b 2024-04-22T15:43:35.515 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (108s) 17s ago 34m 48.0M 3248M 19.0.0-3223-g5883d494 3d983c26e278 9e049dd4678d 2024-04-22T15:43:35.515 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (73s) 17s ago 34m 47.2M 3248M 19.0.0-3223-g5883d494 3d983c26e278 951a8da56333 2024-04-22T15:43:35.516 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (10m) 17s ago 33m 77.0M - 2.43.0 a07b618ecd1d 51cf4df6e8d4 2024-04-22T15:43:35.516 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (26s) 16s ago 31m 76.6M - 19.0.0-3223-g5883d494 3d983c26e278 a4a0fbef80df 2024-04-22T15:43:35.516 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (20s) 17s ago 31m 73.8M - 19.0.0-3223-g5883d494 3d983c26e278 6f2627922357 2024-04-22T15:43:36.370 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph versions | jq -e '"'"'.rgw | length == 1'"'"'' 2024-04-22T15:43:37.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:36 smithi145 ceph-mon[82523]: from='client.44763 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:37.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:36 smithi145 ceph-mon[82523]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 3 op/s 2024-04-22T15:43:37.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:36 smithi145 ceph-mon[88162]: from='client.44763 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:37.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:36 smithi145 ceph-mon[88162]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 3 op/s 2024-04-22T15:43:37.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:36 smithi172 ceph-mon[65307]: from='client.44763 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:37.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:36 smithi172 ceph-mon[65307]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 3 op/s 2024-04-22T15:43:38.658 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:43:39.336 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:38 smithi145 ceph-mon[88162]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 2.8 KiB/s rd, 2 op/s 2024-04-22T15:43:39.336 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:38 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/708494565' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:39.336 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:38 smithi145 ceph-mon[82523]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 2.8 KiB/s rd, 2 op/s 2024-04-22T15:43:39.336 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:38 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/708494565' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:39.362 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph versions | jq -e '"'"'.rgw | keys'"'"' | grep $sha1' 2024-04-22T15:43:39.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:38 smithi172 ceph-mon[65307]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 2.8 KiB/s rd, 2 op/s 2024-04-22T15:43:39.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:38 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/708494565' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:41.206 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:40 smithi145 ceph-mon[82523]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-04-22T15:43:41.206 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:40 smithi145 ceph-mon[88162]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-04-22T15:43:41.269 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:40 smithi172 ceph-mon[65307]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-04-22T15:43:41.514 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)" 2024-04-22T15:43:42.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:41 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:43:42.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:41 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/880886690' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:42.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:41 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:43:42.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:41 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/880886690' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:42.201 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch upgrade status' 2024-04-22T15:43:42.285 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:41 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:43:42.285 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:41 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/880886690' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:43.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:42 smithi145 ceph-mon[82523]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:43:43.122 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:42 smithi145 ceph-mon[88162]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:43:43.426 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:42 smithi172 ceph-mon[65307]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:43:44.272 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:43:44.272 INFO:teuthology.orchestra.run.smithi145.stdout: "target_image": null, 2024-04-22T15:43:44.273 INFO:teuthology.orchestra.run.smithi145.stdout: "in_progress": false, 2024-04-22T15:43:44.273 INFO:teuthology.orchestra.run.smithi145.stdout: "which": "", 2024-04-22T15:43:44.273 INFO:teuthology.orchestra.run.smithi145.stdout: "services_complete": [], 2024-04-22T15:43:44.273 INFO:teuthology.orchestra.run.smithi145.stdout: "progress": null, 2024-04-22T15:43:44.273 INFO:teuthology.orchestra.run.smithi145.stdout: "message": "", 2024-04-22T15:43:44.273 INFO:teuthology.orchestra.run.smithi145.stdout: "is_paused": false 2024-04-22T15:43:44.273 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:43:45.044 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph health detail' 2024-04-22T15:43:45.290 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:44 smithi145 ceph-mon[82523]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:43:45.291 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:44 smithi145 ceph-mon[82523]: from='client.44781 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:45.291 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:44 smithi145 ceph-mon[88162]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:43:45.291 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:44 smithi145 ceph-mon[88162]: from='client.44781 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:45.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:44 smithi172 ceph-mon[65307]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:43:45.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:44 smithi172 ceph-mon[65307]: from='client.44781 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:45.597 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:43:45] "GET /metrics HTTP/1.1" 200 38422 "" "Prometheus/2.43.0" 2024-04-22T15:43:47.174 INFO:teuthology.orchestra.run.smithi145.stdout:HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:43:47.175 INFO:teuthology.orchestra.run.smithi145.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:43:47.175 INFO:teuthology.orchestra.run.smithi145.stdout: daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state 2024-04-22T15:43:47.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:46 smithi145 ceph-mon[82523]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:43:47.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:46 smithi145 ceph-mon[88162]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:43:47.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:46 smithi172 ceph-mon[65307]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:43:47.994 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-22T15:43:48.300 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:47 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/3885171694' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:43:48.300 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:47 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/3885171694' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:43:48.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:48 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/3885171694' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:43:49.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:49 smithi145 ceph-mon[88162]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:43:49.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:49 smithi145 ceph-mon[82523]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:43:49.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:49 smithi172 ceph-mon[65307]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:43:50.153 INFO:teuthology.orchestra.run.smithi145.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:43:50.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:50 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:43:50.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:50 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:43:50.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:50 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:43:50.940 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-22T15:43:50.950 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi145.front.sepia.ceph.com 2024-04-22T15:43:50.951 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-04-22T15:43:51.168 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:51 smithi145 ceph-mon[88162]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:43:51.168 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:51 smithi145 ceph-mon[88162]: from='client.54727 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:51.168 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:51 smithi145 ceph-mon[88162]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:43:51.168 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:51 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:51.168 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:51 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:43:51.168 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:51 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:51.168 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:51 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:43:51.168 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:51 smithi145 ceph-mon[88162]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 997 B/s rd, 0 op/s 2024-04-22T15:43:51.168 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:51 smithi145 ceph-mon[88162]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:43:51.168 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:51 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:51.168 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:51 smithi145 ceph-mon[88162]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:43:51.168 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:51 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:43:51.168 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:51 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:51.169 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:51 smithi145 ceph-mon[88162]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:43:51.169 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:51 smithi145 ceph-mon[82523]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:43:51.169 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:51 smithi145 ceph-mon[82523]: from='client.54727 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:51.169 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:51 smithi145 ceph-mon[82523]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:43:51.169 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:51 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:51.169 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:51 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:43:51.169 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:51 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:51.169 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:51 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:43:51.169 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:51 smithi145 ceph-mon[82523]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 997 B/s rd, 0 op/s 2024-04-22T15:43:51.169 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:51 smithi145 ceph-mon[82523]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:43:51.170 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:51 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:51.170 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:51 smithi145 ceph-mon[82523]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:43:51.170 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:51 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:43:51.170 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:51 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:51.170 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:51 smithi145 ceph-mon[82523]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:43:51.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:51 smithi172 ceph-mon[65307]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:43:51.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:51 smithi172 ceph-mon[65307]: from='client.54727 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:51.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:51 smithi172 ceph-mon[65307]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:43:51.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:51 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:51.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:51 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:43:51.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:51 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:51.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:51 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:43:51.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:51 smithi172 ceph-mon[65307]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 997 B/s rd, 0 op/s 2024-04-22T15:43:51.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:51 smithi172 ceph-mon[65307]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:43:51.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:51 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:51.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:51 smithi172 ceph-mon[65307]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:43:51.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:51 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:43:51.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:51 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:51.469 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:51 smithi172 ceph-mon[65307]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:43:52.467 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:52 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:43:52.590 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:52 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:43:52.590 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:52 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:43:53.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:53 smithi172 ceph-mon[65307]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 619 B/s rd, 0 op/s 2024-04-22T15:43:53.491 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-mon[88162]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 619 B/s rd, 0 op/s 2024-04-22T15:43:53.492 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-mon[82523]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 619 B/s rd, 0 op/s 2024-04-22T15:43:53.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:43:53.541+0000 7f17b18ae700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:53.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:43:53.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:53.873 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:53.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:53.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:53.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:53.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:43:53.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:53.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:43:53.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:53.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:43:53.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:53.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:53.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:53.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:53.874 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:53.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:53.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:53.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:53.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:53.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:43:53.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:43:53.875 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:43:53.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:43:53.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:43:53.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:43:53.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:43:53.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:43:53.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return future.result(timeout) 2024-04-22T15:43:53.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:43:53.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return self.__get_result() 2024-04-22T15:43:53.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:43:53.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: raise self._exception 2024-04-22T15:43:53.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:43:53.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: use_current_daemon_image=reconfig, 2024-04-22T15:43:53.876 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:43:53.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:43:53.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:53.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Creating ceph-iscsi config... 2024-04-22T15:43:53.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:53.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:53.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:53.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:53.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:53.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: Traceback (most recent call last): 2024-04-22T15:43:53.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:53.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: return _run_code(code, main_globals, None, 2024-04-22T15:43:53.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:53.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: exec(code, run_globals) 2024-04-22T15:43:53.877 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:53.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:53.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:53.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:53.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:53.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:53.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:53.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:53.878 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:53.988 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:43:54.702 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:43:54.702 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (10m) 35s ago 32m 27.0M - 0.25.0 c8568f914cd2 1c58ccf37ffd 2024-04-22T15:43:54.702 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (10m) 36s ago 32m 76.5M - 9.4.12 2bacad6d85d8 2f825d15ab10 2024-04-22T15:43:54.702 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 unknown 35s ago 32m 60.9M - 3.5 e1d6a67b021e 9704631ffa0e 2024-04-22T15:43:54.702 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443,9283,8765 running (12m) 36s ago 37m 454M - 19.0.0-3223-g5883d494 3d983c26e278 e7500ef71604 2024-04-22T15:43:54.702 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (22m) 35s ago 39m 516M - 19.0.0-3223-g5883d494 3d983c26e278 b2de97566cf4 2024-04-22T15:43:54.702 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (8m) 35s ago 39m 62.2M 2048M 19.0.0-3223-g5883d494 3d983c26e278 7bfdc11120f4 2024-04-22T15:43:54.702 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (9m) 36s ago 37m 54.7M 2048M 19.0.0-3223-g5883d494 3d983c26e278 0a360b08a6d6 2024-04-22T15:43:54.702 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (7m) 35s ago 38m 48.4M 2048M 19.0.0-3223-g5883d494 3d983c26e278 38dcfa677c18 2024-04-22T15:43:54.702 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (11m) 35s ago 33m 15.5M - 1.5.0 0da6a335fe13 8c75c26fec05 2024-04-22T15:43:54.703 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (11m) 36s ago 33m 16.0M - 1.5.0 0da6a335fe13 8fd2949e2eaf 2024-04-22T15:43:54.703 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (6m) 35s ago 37m 60.3M 2992M 19.0.0-3223-g5883d494 3d983c26e278 e4bc06317af0 2024-04-22T15:43:54.703 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (5m) 35s ago 36m 52.3M 2992M 19.0.0-3223-g5883d494 3d983c26e278 d299f622f08d 2024-04-22T15:43:54.703 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (4m) 35s ago 36m 48.5M 2992M 19.0.0-3223-g5883d494 3d983c26e278 6aea4a926fe4 2024-04-22T15:43:54.703 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (3m) 35s ago 36m 55.1M 2992M 19.0.0-3223-g5883d494 3d983c26e278 c0cd2bdc1439 2024-04-22T15:43:54.703 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (3m) 36s ago 35m 51.4M 3248M 19.0.0-3223-g5883d494 3d983c26e278 7ac5776bc07a 2024-04-22T15:43:54.703 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (2m) 36s ago 35m 57.0M 3248M 19.0.0-3223-g5883d494 3d983c26e278 5f803668a17b 2024-04-22T15:43:54.703 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (2m) 36s ago 35m 48.0M 3248M 19.0.0-3223-g5883d494 3d983c26e278 9e049dd4678d 2024-04-22T15:43:54.703 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (92s) 36s ago 34m 47.2M 3248M 19.0.0-3223-g5883d494 3d983c26e278 951a8da56333 2024-04-22T15:43:54.703 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (11m) 36s ago 33m 77.0M - 2.43.0 a07b618ecd1d 51cf4df6e8d4 2024-04-22T15:43:54.703 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (45s) 35s ago 32m 76.6M - 19.0.0-3223-g5883d494 3d983c26e278 a4a0fbef80df 2024-04-22T15:43:54.703 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (40s) 36s ago 32m 73.8M - 19.0.0-3223-g5883d494 3d983c26e278 6f2627922357 2024-04-22T15:43:55.118 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:43:55.119 INFO:teuthology.orchestra.run.smithi145.stdout: "mon": { 2024-04-22T15:43:55.119 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 3 2024-04-22T15:43:55.119 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:43:55.119 INFO:teuthology.orchestra.run.smithi145.stdout: "mgr": { 2024-04-22T15:43:55.119 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:43:55.119 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:43:55.119 INFO:teuthology.orchestra.run.smithi145.stdout: "osd": { 2024-04-22T15:43:55.119 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 8 2024-04-22T15:43:55.119 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:43:55.119 INFO:teuthology.orchestra.run.smithi145.stdout: "rgw": { 2024-04-22T15:43:55.119 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:43:55.119 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:43:55.119 INFO:teuthology.orchestra.run.smithi145.stdout: "overall": { 2024-04-22T15:43:55.120 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 15 2024-04-22T15:43:55.120 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:43:55.120 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:43:55.372 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:43:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:43:55] "GET /metrics HTTP/1.1" 200 38415 "" "Prometheus/2.43.0" 2024-04-22T15:43:55.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:55 smithi145 ceph-mon[88162]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:55.373 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:43:55.373 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:55.373 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:55.373 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:55.373 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:55.373 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:55.373 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:43:55.373 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:55.373 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:43:55.373 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:55.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:43:55.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:55.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:55.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:55.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:55.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:55.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:55.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:55.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:55.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:55.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:43:55.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:43:55.374 INFO:journalctl@ceph.mon.c.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:43:55.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:55 smithi145 ceph-mon[82523]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:55.375 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:43:55.375 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:55.375 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:55.375 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:55.375 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:55.375 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:55.375 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:43:55.375 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:55.375 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:43:55.375 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:55.376 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:43:55.376 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:55.376 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:55.376 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:55.376 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:55.376 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:55.376 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:55.376 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:55.376 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:55.376 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:55.376 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:43:55.376 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:43:55.376 INFO:journalctl@ceph.mon.a.smithi145.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:43:55.377 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:43:55.377 INFO:journalctl@ceph.mon.c.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:43:55.377 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:43:55.377 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:43:55.377 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:43:55.377 INFO:journalctl@ceph.mon.c.smithi145.stdout: return future.result(timeout) 2024-04-22T15:43:55.377 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:43:55.377 INFO:journalctl@ceph.mon.c.smithi145.stdout: return self.__get_result() 2024-04-22T15:43:55.377 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:43:55.377 INFO:journalctl@ceph.mon.c.smithi145.stdout: raise self._exception 2024-04-22T15:43:55.377 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:43:55.377 INFO:journalctl@ceph.mon.c.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:43:55.377 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:43:55.378 INFO:journalctl@ceph.mon.c.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:43:55.378 INFO:journalctl@ceph.mon.c.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:55.378 INFO:journalctl@ceph.mon.c.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:43:55.378 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:55.378 INFO:journalctl@ceph.mon.c.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:55.378 INFO:journalctl@ceph.mon.c.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:55.378 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:55.378 INFO:journalctl@ceph.mon.c.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:55.378 INFO:journalctl@ceph.mon.c.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:43:55.378 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:55.378 INFO:journalctl@ceph.mon.c.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:43:55.378 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:55.378 INFO:journalctl@ceph.mon.c.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:43:55.378 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:55.379 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:55.379 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:55.379 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:55.379 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:43:55.379 INFO:journalctl@ceph.mon.a.smithi145.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:43:55.379 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:43:55.379 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:43:55.379 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:43:55.379 INFO:journalctl@ceph.mon.a.smithi145.stdout: return future.result(timeout) 2024-04-22T15:43:55.379 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:43:55.379 INFO:journalctl@ceph.mon.a.smithi145.stdout: return self.__get_result() 2024-04-22T15:43:55.380 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:43:55.380 INFO:journalctl@ceph.mon.a.smithi145.stdout: raise self._exception 2024-04-22T15:43:55.380 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:43:55.380 INFO:journalctl@ceph.mon.a.smithi145.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:43:55.380 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:43:55.380 INFO:journalctl@ceph.mon.a.smithi145.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:43:55.380 INFO:journalctl@ceph.mon.a.smithi145.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:55.380 INFO:journalctl@ceph.mon.a.smithi145.stdout: Creating ceph-iscsi config... 2024-04-22T15:43:55.380 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:55.380 INFO:journalctl@ceph.mon.a.smithi145.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:55.380 INFO:journalctl@ceph.mon.a.smithi145.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:55.380 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:55.380 INFO:journalctl@ceph.mon.a.smithi145.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:55.380 INFO:journalctl@ceph.mon.a.smithi145.stdout: Traceback (most recent call last): 2024-04-22T15:43:55.380 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:55.381 INFO:journalctl@ceph.mon.a.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:43:55.381 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:55.381 INFO:journalctl@ceph.mon.a.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:43:55.381 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:55.381 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:55.381 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:55.381 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:55.381 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:55.381 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:55.381 INFO:journalctl@ceph.mon.c.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:55.381 INFO:journalctl@ceph.mon.c.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:55.382 INFO:journalctl@ceph.mon.c.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:55.382 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:55 smithi145 ceph-mon[88162]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:43:55.382 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:55 smithi145 ceph-mon[88162]: from='client.44799 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:55.382 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:55 smithi145 ceph-mon[88162]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:43:55.382 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:55 smithi145 ceph-mon[88162]: from='client.54739 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:55.382 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:55 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/4263543454' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:55.382 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:55.382 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:55.382 INFO:journalctl@ceph.mon.a.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:55.382 INFO:journalctl@ceph.mon.a.smithi145.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:55.382 INFO:journalctl@ceph.mon.a.smithi145.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:55.382 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:55 smithi145 ceph-mon[82523]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:43:55.383 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:55 smithi145 ceph-mon[82523]: from='client.44799 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:55.383 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:55 smithi145 ceph-mon[82523]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:43:55.383 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:55 smithi145 ceph-mon[82523]: from='client.54739 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:55.383 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:55 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/4263543454' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:55.470 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:43:55.470 INFO:teuthology.orchestra.run.smithi145.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa", 2024-04-22T15:43:55.470 INFO:teuthology.orchestra.run.smithi145.stdout: "in_progress": true, 2024-04-22T15:43:55.470 INFO:teuthology.orchestra.run.smithi145.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T15:43:55.470 INFO:teuthology.orchestra.run.smithi145.stdout: "services_complete": [], 2024-04-22T15:43:55.470 INFO:teuthology.orchestra.run.smithi145.stdout: "progress": "", 2024-04-22T15:43:55.470 INFO:teuthology.orchestra.run.smithi145.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa image", 2024-04-22T15:43:55.470 INFO:teuthology.orchestra.run.smithi145.stdout: "is_paused": false 2024-04-22T15:43:55.470 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:43:55.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:55 smithi172 ceph-mon[65307]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:55.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:43:55.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:55.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:55.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:55.718 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:55.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:55.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:43:55.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:55.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:43:55.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:55.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:43:55.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:55.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:55.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:55.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:55.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:55.719 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:55.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:55.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:55.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:55.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:43:55.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-22T15:43:55.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-22T15:43:55.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-22T15:43:55.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-22T15:43:55.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-22T15:43:55.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-22T15:43:55.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-22T15:43:55.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return future.result(timeout) 2024-04-22T15:43:55.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-22T15:43:55.720 INFO:journalctl@ceph.mon.b.smithi172.stdout: return self.__get_result() 2024-04-22T15:43:55.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-22T15:43:55.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: raise self._exception 2024-04-22T15:43:55.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-22T15:43:55.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: use_current_daemon_image=reconfig, 2024-04-22T15:43:55.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1709, in _run_cephadm 2024-04-22T15:43:55.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-22T15:43:55.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:43:55.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Creating ceph-iscsi config... 2024-04-22T15:43:55.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:43:55.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:43:55.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: Non-zero exit code 1 from systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:55.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:55.721 INFO:journalctl@ceph.mon.b.smithi172.stdout: systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:55.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: Traceback (most recent call last): 2024-04-22T15:43:55.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:43:55.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:43:55.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:43:55.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: exec(code, run_globals) 2024-04-22T15:43:55.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:43:55.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:43:55.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:43:55.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:43:55.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:43:55.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1123, in deploy_daemon 2024-04-22T15:43:55.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:43:55.722 INFO:journalctl@ceph.mon.b.smithi172.stdout: RuntimeError: Failed command: systemctl restart ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:43:55.723 INFO:journalctl@ceph.mon.b.smithi172.stdout: See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:43:55.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:55 smithi172 ceph-mon[65307]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5883d494d3973b30b3f03e4a3f0e1e06e3c131aa 2024-04-22T15:43:55.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:55 smithi172 ceph-mon[65307]: from='client.44799 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:55.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:55 smithi172 ceph-mon[65307]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:43:55.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:55 smithi172 ceph-mon[65307]: from='client.54739 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:55.723 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:55 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/4263543454' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:55.870 INFO:teuthology.orchestra.run.smithi145.stdout:HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:43:55.870 INFO:teuthology.orchestra.run.smithi145.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] 2024-04-22T15:43:55.871 INFO:teuthology.orchestra.run.smithi145.stdout: daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state 2024-04-22T15:43:56.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[82523]: from='client.44811 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:56.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/3001211125' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:43:56.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:56.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:43:56.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:56.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:56.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:56.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:56.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:56.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:56.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:56.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:56.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:56.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:56.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:56.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:56.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:56.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:56.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:56.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:56.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:56.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[88162]: from='client.44811 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:56.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/3001211125' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:43:56.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:56.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:43:56.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:56.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:56.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:56.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:56.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:56.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:56.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:56.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:56.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:56.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:56.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:56.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:56.625 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:56.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:56.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:56.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:56.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:56 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:56.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:56 smithi172 ceph-mon[65307]: from='client.44811 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:56.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:56 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/3001211125' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:43:56.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:56 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:56.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:56 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:43:56.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:56 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:56.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:56 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:56.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:56 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:56.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:56 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:56.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:56 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:56.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:56 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:56.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:56 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:56.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:56 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:56.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:56 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:56.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:56 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:56.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:56 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:56.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:56 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:56.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:56 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:56.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:56 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:56.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:56 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:56.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:56 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:43:56.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:56 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:57.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:57 smithi145 ceph-mon[82523]: from='client.54757 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:57.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:57 smithi145 ceph-mon[82523]: Upgrade: Target is version 19.0.0-3223-g5883d494 (squid) 2024-04-22T15:43:57.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:57 smithi145 ceph-mon[82523]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, digests ['quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb'] 2024-04-22T15:43:57.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:57 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all mgr 2024-04-22T15:43:57.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:57 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all mon 2024-04-22T15:43:57.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:57 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all crash 2024-04-22T15:43:57.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:57 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all osd 2024-04-22T15:43:57.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:57 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all mds 2024-04-22T15:43:57.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:57 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all rgw 2024-04-22T15:43:57.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:57 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T15:43:57.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:57 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-22T15:43:57.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:57 smithi145 ceph-mon[82523]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T15:43:57.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:57 smithi145 ceph-mon[82523]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:43:57.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:57 smithi145 ceph-mon[88162]: from='client.54757 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:57.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:57 smithi145 ceph-mon[88162]: Upgrade: Target is version 19.0.0-3223-g5883d494 (squid) 2024-04-22T15:43:57.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:57 smithi145 ceph-mon[88162]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, digests ['quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb'] 2024-04-22T15:43:57.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:57 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all mgr 2024-04-22T15:43:57.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:57 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all mon 2024-04-22T15:43:57.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:57 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all crash 2024-04-22T15:43:57.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:57 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all osd 2024-04-22T15:43:57.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:57 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all mds 2024-04-22T15:43:57.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:57 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all rgw 2024-04-22T15:43:57.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:57 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T15:43:57.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:57 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-22T15:43:57.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:57 smithi145 ceph-mon[88162]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T15:43:57.374 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:57 smithi145 ceph-mon[88162]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:43:57.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:57 smithi172 ceph-mon[65307]: from='client.54757 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:43:57.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:57 smithi172 ceph-mon[65307]: Upgrade: Target is version 19.0.0-3223-g5883d494 (squid) 2024-04-22T15:43:57.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:57 smithi172 ceph-mon[65307]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, digests ['quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb'] 2024-04-22T15:43:57.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:57 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all mgr 2024-04-22T15:43:57.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:57 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all mon 2024-04-22T15:43:57.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:57 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all crash 2024-04-22T15:43:57.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:57 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all osd 2024-04-22T15:43:57.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:57 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all mds 2024-04-22T15:43:57.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:57 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all rgw 2024-04-22T15:43:57.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:57 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all rbd-mirror 2024-04-22T15:43:57.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:57 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-22T15:43:57.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:57 smithi172 ceph-mon[65307]: Upgrade: Setting container_image for all ceph-exporter 2024-04-22T15:43:57.719 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:57 smithi172 ceph-mon[65307]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:43:59.122 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:58 smithi145 ceph-mon[88162]: Upgrade: Updating iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:59.122 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:58 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:59.122 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:58 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:43:59.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:58 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:59.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:58 smithi145 ceph-mon[88162]: Deploying daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:43:59.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:43:58 smithi145 ceph-mon[88162]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 619 B/s rd, 0 op/s 2024-04-22T15:43:59.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:58 smithi145 ceph-mon[82523]: Upgrade: Updating iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:59.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:58 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:59.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:58 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:43:59.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:58 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:59.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:58 smithi145 ceph-mon[82523]: Deploying daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:43:59.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:43:58 smithi145 ceph-mon[82523]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 619 B/s rd, 0 op/s 2024-04-22T15:43:59.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:58 smithi172 ceph-mon[65307]: Upgrade: Updating iscsi.foo.smithi145.ljpvqb 2024-04-22T15:43:59.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:58 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:43:59.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:58 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:43:59.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:58 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:43:59.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:58 smithi172 ceph-mon[65307]: Deploying daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:43:59.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:43:58 smithi172 ceph-mon[65307]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 619 B/s rd, 0 op/s 2024-04-22T15:44:01.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:01 smithi145 ceph-mon[82523]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:44:01.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:01 smithi145 ceph-mon[88162]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:44:01.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:01 smithi172 ceph-mon[65307]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-22T15:44:02.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:02 smithi145 ceph-mon[82523]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:44:02.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:02 smithi145 ceph-mon[88162]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:44:02.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:02 smithi172 ceph-mon[65307]: from='client.25249 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:44:03.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:03 smithi145 ceph-mon[82523]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:44:03.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:03 smithi145 ceph-mon[88162]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:44:03.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:03 smithi172 ceph-mon[65307]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:44:05.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:05 smithi145 ceph-mon[88162]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:44:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:05 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:44:05.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:05 smithi145 ceph-mon[82523]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:44:05.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:05 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:44:05.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:44:05 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:44:05] "GET /metrics HTTP/1.1" 200 38422 "" "Prometheus/2.43.0" 2024-04-22T15:44:05.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:05 smithi172 ceph-mon[65307]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:44:05.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:05 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:44:07.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:07 smithi145 ceph-mon[82523]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:44:07.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:07 smithi145 ceph-mon[88162]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:44:07.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:07 smithi172 ceph-mon[65307]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:44:09.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:09 smithi145 ceph-mon[82523]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:44:09.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:09 smithi145 ceph-mon[88162]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:44:09.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:09 smithi172 ceph-mon[65307]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:44:11.190 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:44:10 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:44:10.894+0000 7f17b18ae700 -1 log_channel(cephadm) log [ERR] : Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi145.ljpvqb on host smithi145 failed. 2024-04-22T15:44:11.455 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:11 smithi145 ceph-mon[88162]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:44:11.455 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:11 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:44:11.455 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:11 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:44:11.455 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:11 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:44:11.455 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:11 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:44:11.455 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:11 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:44:11.455 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:11 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:44:11.455 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:11 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:44:11.455 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:11 smithi145 ceph-mon[82523]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:44:11.456 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:11 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:44:11.456 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:11 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:44:11.456 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:11 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:44:11.456 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:11 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:44:11.456 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:11 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:44:11.456 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:11 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:44:11.456 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:11 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:44:11.520 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:11 smithi172 ceph-mon[65307]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:44:11.520 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:11 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:44:11.520 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:11 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:44:11.520 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:11 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:44:11.520 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:11 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:44:11.520 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:11 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:44:11.521 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:11 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi145.ljpvqb", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T15:44:11.521 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:11 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:44:12.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:12 smithi145 ceph-mon[82523]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi145.ljpvqb on host smithi145 failed. 2024-04-22T15:44:12.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:12 smithi145 ceph-mon[82523]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 955 B/s rd, 0 op/s 2024-04-22T15:44:12.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:12 smithi145 ceph-mon[82523]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:44:12.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:12 smithi145 ceph-mon[82523]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 762 B/s rd, 0 op/s 2024-04-22T15:44:12.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:12 smithi145 ceph-mon[82523]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:44:12.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:12 smithi145 ceph-mon[82523]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:44:12.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:12 smithi145 ceph-mon[82523]: Health check failed: Upgrading daemon iscsi.foo.smithi145.ljpvqb on host smithi145 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-04-22T15:44:12.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:12 smithi145 ceph-mon[88162]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi145.ljpvqb on host smithi145 failed. 2024-04-22T15:44:12.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:12 smithi145 ceph-mon[88162]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 955 B/s rd, 0 op/s 2024-04-22T15:44:12.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:12 smithi145 ceph-mon[88162]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:44:12.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:12 smithi145 ceph-mon[88162]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 762 B/s rd, 0 op/s 2024-04-22T15:44:12.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:12 smithi145 ceph-mon[88162]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:44:12.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:12 smithi145 ceph-mon[88162]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:44:12.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:12 smithi145 ceph-mon[88162]: Health check failed: Upgrading daemon iscsi.foo.smithi145.ljpvqb on host smithi145 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-04-22T15:44:12.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:12 smithi172 ceph-mon[65307]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi145.ljpvqb on host smithi145 failed. 2024-04-22T15:44:12.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:12 smithi172 ceph-mon[65307]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 955 B/s rd, 0 op/s 2024-04-22T15:44:12.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:12 smithi172 ceph-mon[65307]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:44:12.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:12 smithi172 ceph-mon[65307]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 762 B/s rd, 0 op/s 2024-04-22T15:44:12.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:12 smithi172 ceph-mon[65307]: Reconfiguring iscsi.foo.smithi145.ljpvqb (dependencies changed)... 2024-04-22T15:44:12.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:12 smithi172 ceph-mon[65307]: Reconfiguring daemon iscsi.foo.smithi145.ljpvqb on smithi145 2024-04-22T15:44:12.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:12 smithi172 ceph-mon[65307]: Health check failed: Upgrading daemon iscsi.foo.smithi145.ljpvqb on host smithi145 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-04-22T15:44:14.372 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:14 smithi145 ceph-mon[88162]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 762 B/s rd, 0 op/s 2024-04-22T15:44:14.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:14 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:44:14.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:14 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:44:14.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:14 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T15:44:14.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:14 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T15:44:14.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:14 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:44:14.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:14 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/972663568' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T15:44:14.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:14 smithi145 ceph-mon[82523]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 762 B/s rd, 0 op/s 2024-04-22T15:44:14.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:14 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:44:14.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:14 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:44:14.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:14 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T15:44:14.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:14 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T15:44:14.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:14 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:44:14.374 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:14 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/972663568' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T15:44:14.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:14 smithi172 ceph-mon[65307]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 762 B/s rd, 0 op/s 2024-04-22T15:44:14.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:14 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:44:14.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:14 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:44:14.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:14 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T15:44:14.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:14 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T15:44:14.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:14 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:44:14.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:14 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/972663568' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T15:44:15.543 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:15 smithi145 ceph-mon[88162]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T15:44:15.543 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:15 smithi145 ceph-mon[88162]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T15:44:15.543 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:15 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/169774111' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2327315169"}]: dispatch 2024-04-22T15:44:15.544 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:15 smithi145 ceph-mon[82523]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T15:44:15.544 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:15 smithi145 ceph-mon[82523]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T15:44:15.544 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:15 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/169774111' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2327315169"}]: dispatch 2024-04-22T15:44:15.544 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:44:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:44:15] "GET /metrics HTTP/1.1" 200 38422 "" "Prometheus/2.43.0" 2024-04-22T15:44:15.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:15 smithi172 ceph-mon[65307]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T15:44:15.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:15 smithi172 ceph-mon[65307]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T15:44:15.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:15 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/169774111' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2327315169"}]: dispatch 2024-04-22T15:44:16.588 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:16 smithi145 ceph-mon[88162]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:44:16.589 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:16 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/169774111' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2327315169"}]': finished 2024-04-22T15:44:16.589 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:16 smithi145 ceph-mon[88162]: osdmap e150: 8 total, 8 up, 8 in 2024-04-22T15:44:16.589 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:16 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/2285946571' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1396135830"}]: dispatch 2024-04-22T15:44:16.589 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:16 smithi145 ceph-mon[88162]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1396135830"}]: dispatch 2024-04-22T15:44:16.589 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:16 smithi145 ceph-mon[82523]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:44:16.589 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:16 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/169774111' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2327315169"}]': finished 2024-04-22T15:44:16.589 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:16 smithi145 ceph-mon[82523]: osdmap e150: 8 total, 8 up, 8 in 2024-04-22T15:44:16.589 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:16 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/2285946571' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1396135830"}]: dispatch 2024-04-22T15:44:16.589 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:16 smithi145 ceph-mon[82523]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1396135830"}]: dispatch 2024-04-22T15:44:16.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:16 smithi172 ceph-mon[65307]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:44:16.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:16 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/169774111' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/2327315169"}]': finished 2024-04-22T15:44:16.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:16 smithi172 ceph-mon[65307]: osdmap e150: 8 total, 8 up, 8 in 2024-04-22T15:44:16.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:16 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/2285946571' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1396135830"}]: dispatch 2024-04-22T15:44:16.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:16 smithi172 ceph-mon[65307]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1396135830"}]: dispatch 2024-04-22T15:44:17.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:17 smithi145 ceph-mon[82523]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1396135830"}]': finished 2024-04-22T15:44:17.372 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:17 smithi145 ceph-mon[82523]: osdmap e151: 8 total, 8 up, 8 in 2024-04-22T15:44:17.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:17 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/1996626615' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/576893106"}]: dispatch 2024-04-22T15:44:17.373 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:17 smithi145 ceph-mon[82523]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/576893106"}]: dispatch 2024-04-22T15:44:17.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:17 smithi145 ceph-mon[88162]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1396135830"}]': finished 2024-04-22T15:44:17.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:17 smithi145 ceph-mon[88162]: osdmap e151: 8 total, 8 up, 8 in 2024-04-22T15:44:17.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:17 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/1996626615' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/576893106"}]: dispatch 2024-04-22T15:44:17.373 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:17 smithi145 ceph-mon[88162]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/576893106"}]: dispatch 2024-04-22T15:44:17.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:17 smithi172 ceph-mon[65307]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1396135830"}]': finished 2024-04-22T15:44:17.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:17 smithi172 ceph-mon[65307]: osdmap e151: 8 total, 8 up, 8 in 2024-04-22T15:44:17.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:17 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/1996626615' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/576893106"}]: dispatch 2024-04-22T15:44:17.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:17 smithi172 ceph-mon[65307]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/576893106"}]: dispatch 2024-04-22T15:44:18.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:18 smithi145 ceph-mon[88162]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:44:18.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:18 smithi145 ceph-mon[88162]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/576893106"}]': finished 2024-04-22T15:44:18.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:18 smithi145 ceph-mon[88162]: osdmap e152: 8 total, 8 up, 8 in 2024-04-22T15:44:18.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:18 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:44:18.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:18 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:44:18.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:18 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/2321860779' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1128652857"}]: dispatch 2024-04-22T15:44:18.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:18 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:44:18.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:18 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:44:18.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:18 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:44:18.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:18 smithi145 ceph-mon[82523]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:44:18.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:18 smithi145 ceph-mon[82523]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/576893106"}]': finished 2024-04-22T15:44:18.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:18 smithi145 ceph-mon[82523]: osdmap e152: 8 total, 8 up, 8 in 2024-04-22T15:44:18.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:18 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:44:18.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:18 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:44:18.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:18 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/2321860779' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1128652857"}]: dispatch 2024-04-22T15:44:18.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:18 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:44:18.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:18 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:44:18.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:18 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:44:18.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:18 smithi172 ceph-mon[65307]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-22T15:44:18.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:18 smithi172 ceph-mon[65307]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/576893106"}]': finished 2024-04-22T15:44:18.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:18 smithi172 ceph-mon[65307]: osdmap e152: 8 total, 8 up, 8 in 2024-04-22T15:44:18.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:18 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:44:18.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:18 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:44:18.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:18 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/2321860779' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1128652857"}]: dispatch 2024-04-22T15:44:18.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:18 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:44:18.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:18 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:44:18.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:18 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:44:19.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:19 smithi145 ceph-mon[88162]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 430 B/s rd, 0 op/s 2024-04-22T15:44:19.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:19 smithi145 ceph-mon[88162]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state']) 2024-04-22T15:44:19.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:19 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/2321860779' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1128652857"}]': finished 2024-04-22T15:44:19.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:19 smithi145 ceph-mon[88162]: osdmap e153: 8 total, 8 up, 8 in 2024-04-22T15:44:19.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:19 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/2167322845' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3699087930"}]: dispatch 2024-04-22T15:44:19.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:19 smithi145 ceph-mon[82523]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 430 B/s rd, 0 op/s 2024-04-22T15:44:19.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:19 smithi145 ceph-mon[82523]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state']) 2024-04-22T15:44:19.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:19 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/2321860779' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1128652857"}]': finished 2024-04-22T15:44:19.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:19 smithi145 ceph-mon[82523]: osdmap e153: 8 total, 8 up, 8 in 2024-04-22T15:44:19.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:19 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/2167322845' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3699087930"}]: dispatch 2024-04-22T15:44:19.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:19 smithi172 ceph-mon[65307]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 430 B/s rd, 0 op/s 2024-04-22T15:44:19.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:19 smithi172 ceph-mon[65307]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state']) 2024-04-22T15:44:19.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:19 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/2321860779' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1128652857"}]': finished 2024-04-22T15:44:19.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:19 smithi172 ceph-mon[65307]: osdmap e153: 8 total, 8 up, 8 in 2024-04-22T15:44:19.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:19 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/2167322845' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3699087930"}]: dispatch 2024-04-22T15:44:20.535 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:20 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:44:20.535 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:20 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:44:20.535 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:20 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/2167322845' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3699087930"}]': finished 2024-04-22T15:44:20.535 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:20 smithi145 ceph-mon[88162]: osdmap e154: 8 total, 8 up, 8 in 2024-04-22T15:44:20.535 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:20 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/1277780741' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/1372599635"}]: dispatch 2024-04-22T15:44:20.536 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:20 smithi145 ceph-mon[88162]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/1372599635"}]: dispatch 2024-04-22T15:44:20.536 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:20 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:44:20.536 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:20 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:44:20.536 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:20 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/2167322845' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3699087930"}]': finished 2024-04-22T15:44:20.536 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:20 smithi145 ceph-mon[82523]: osdmap e154: 8 total, 8 up, 8 in 2024-04-22T15:44:20.536 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:20 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/1277780741' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/1372599635"}]: dispatch 2024-04-22T15:44:20.536 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:20 smithi145 ceph-mon[82523]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/1372599635"}]: dispatch 2024-04-22T15:44:20.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:20 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:44:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:20 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:44:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:20 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/2167322845' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3699087930"}]': finished 2024-04-22T15:44:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:20 smithi172 ceph-mon[65307]: osdmap e154: 8 total, 8 up, 8 in 2024-04-22T15:44:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:20 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/1277780741' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/1372599635"}]: dispatch 2024-04-22T15:44:20.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:20 smithi172 ceph-mon[65307]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/1372599635"}]: dispatch 2024-04-22T15:44:21.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:21 smithi145 ceph-mon[82523]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:44:21.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:21 smithi145 ceph-mon[82523]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/1372599635"}]': finished 2024-04-22T15:44:21.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:21 smithi145 ceph-mon[82523]: osdmap e155: 8 total, 8 up, 8 in 2024-04-22T15:44:21.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:21 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/2195065092' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/1372599635"}]: dispatch 2024-04-22T15:44:21.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:21 smithi145 ceph-mon[88162]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:44:21.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:21 smithi145 ceph-mon[88162]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/1372599635"}]': finished 2024-04-22T15:44:21.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:21 smithi145 ceph-mon[88162]: osdmap e155: 8 total, 8 up, 8 in 2024-04-22T15:44:21.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:21 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/2195065092' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/1372599635"}]: dispatch 2024-04-22T15:44:21.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:21 smithi172 ceph-mon[65307]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:44:21.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:21 smithi172 ceph-mon[65307]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/1372599635"}]': finished 2024-04-22T15:44:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:21 smithi172 ceph-mon[65307]: osdmap e155: 8 total, 8 up, 8 in 2024-04-22T15:44:21.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:21 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/2195065092' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/1372599635"}]: dispatch 2024-04-22T15:44:22.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:22 smithi172 ceph-mon[65307]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:44:22.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:22 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/2195065092' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/1372599635"}]': finished 2024-04-22T15:44:22.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:22 smithi172 ceph-mon[65307]: osdmap e156: 8 total, 8 up, 8 in 2024-04-22T15:44:22.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:22 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/4234748830' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/4123337583"}]: dispatch 2024-04-22T15:44:22.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:22 smithi172 ceph-mon[65307]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/4123337583"}]: dispatch 2024-04-22T15:44:23.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:22 smithi145 ceph-mon[88162]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:44:23.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:22 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/2195065092' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/1372599635"}]': finished 2024-04-22T15:44:23.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:22 smithi145 ceph-mon[88162]: osdmap e156: 8 total, 8 up, 8 in 2024-04-22T15:44:23.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:22 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/4234748830' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/4123337583"}]: dispatch 2024-04-22T15:44:23.124 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:22 smithi145 ceph-mon[88162]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/4123337583"}]: dispatch 2024-04-22T15:44:23.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:22 smithi145 ceph-mon[82523]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T15:44:23.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:22 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/2195065092' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/1372599635"}]': finished 2024-04-22T15:44:23.125 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:22 smithi145 ceph-mon[82523]: osdmap e156: 8 total, 8 up, 8 in 2024-04-22T15:44:23.125 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:22 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/4234748830' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/4123337583"}]: dispatch 2024-04-22T15:44:23.125 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:22 smithi145 ceph-mon[82523]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/4123337583"}]: dispatch 2024-04-22T15:44:23.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:23 smithi172 ceph-mon[65307]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/4123337583"}]': finished 2024-04-22T15:44:23.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:23 smithi172 ceph-mon[65307]: osdmap e157: 8 total, 8 up, 8 in 2024-04-22T15:44:23.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:23 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/837022302' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/536309348"}]: dispatch 2024-04-22T15:44:24.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:23 smithi145 ceph-mon[82523]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/4123337583"}]': finished 2024-04-22T15:44:24.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:23 smithi145 ceph-mon[82523]: osdmap e157: 8 total, 8 up, 8 in 2024-04-22T15:44:24.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:23 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/837022302' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/536309348"}]: dispatch 2024-04-22T15:44:24.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:23 smithi145 ceph-mon[88162]: from='client.? ' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/4123337583"}]': finished 2024-04-22T15:44:24.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:23 smithi145 ceph-mon[88162]: osdmap e157: 8 total, 8 up, 8 in 2024-04-22T15:44:24.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:23 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/837022302' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/536309348"}]: dispatch 2024-04-22T15:44:24.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:24 smithi172 ceph-mon[65307]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:44:24.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:24 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/837022302' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/536309348"}]': finished 2024-04-22T15:44:24.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:24 smithi172 ceph-mon[65307]: osdmap e158: 8 total, 8 up, 8 in 2024-04-22T15:44:24.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:24 smithi172 ceph-mon[65307]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:44:24.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:24 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/915759968' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1533922914"}]: dispatch 2024-04-22T15:44:25.122 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:24 smithi145 ceph-mon[88162]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:44:25.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:24 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/837022302' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/536309348"}]': finished 2024-04-22T15:44:25.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:24 smithi145 ceph-mon[88162]: osdmap e158: 8 total, 8 up, 8 in 2024-04-22T15:44:25.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:24 smithi145 ceph-mon[88162]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:44:25.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:24 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/915759968' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1533922914"}]: dispatch 2024-04-22T15:44:25.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:24 smithi145 ceph-mon[82523]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:44:25.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:24 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/837022302' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6801/536309348"}]': finished 2024-04-22T15:44:25.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:24 smithi145 ceph-mon[82523]: osdmap e158: 8 total, 8 up, 8 in 2024-04-22T15:44:25.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:24 smithi145 ceph-mon[82523]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:44:25.124 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:24 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/915759968' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1533922914"}]: dispatch 2024-04-22T15:44:25.622 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:44:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:44:25] "GET /metrics HTTP/1.1" 200 38498 "" "Prometheus/2.43.0" 2024-04-22T15:44:25.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:25 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/915759968' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1533922914"}]': finished 2024-04-22T15:44:25.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:25 smithi172 ceph-mon[65307]: osdmap e159: 8 total, 8 up, 8 in 2024-04-22T15:44:25.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:25 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/859150090' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3214355555"}]: dispatch 2024-04-22T15:44:26.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:25 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/915759968' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1533922914"}]': finished 2024-04-22T15:44:26.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:25 smithi145 ceph-mon[82523]: osdmap e159: 8 total, 8 up, 8 in 2024-04-22T15:44:26.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:25 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/859150090' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3214355555"}]: dispatch 2024-04-22T15:44:26.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:25 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/915759968' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/1533922914"}]': finished 2024-04-22T15:44:26.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:25 smithi145 ceph-mon[88162]: osdmap e159: 8 total, 8 up, 8 in 2024-04-22T15:44:26.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:25 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/859150090' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3214355555"}]: dispatch 2024-04-22T15:44:26.226 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:44:26.561 INFO:teuthology.orchestra.run.smithi145.stdout:"Error: UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi145.ljpvqb on host smithi145 failed." 2024-04-22T15:44:26.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:26 smithi145 ceph-mon[88162]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:44:26.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:26 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/859150090' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3214355555"}]': finished 2024-04-22T15:44:26.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:26 smithi145 ceph-mon[88162]: osdmap e160: 8 total, 8 up, 8 in 2024-04-22T15:44:26.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:26 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/3363684314' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/536309348"}]: dispatch 2024-04-22T15:44:26.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:26 smithi145 ceph-mon[88162]: from='client.54862 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:44:26.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:26 smithi145 ceph-mon[82523]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:44:26.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:26 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/859150090' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3214355555"}]': finished 2024-04-22T15:44:26.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:26 smithi145 ceph-mon[82523]: osdmap e160: 8 total, 8 up, 8 in 2024-04-22T15:44:26.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:26 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/3363684314' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/536309348"}]: dispatch 2024-04-22T15:44:26.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:26 smithi145 ceph-mon[82523]: from='client.54862 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:44:26.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:26 smithi172 ceph-mon[65307]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:44:26.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:26 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/859150090' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:0/3214355555"}]': finished 2024-04-22T15:44:26.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:26 smithi172 ceph-mon[65307]: osdmap e160: 8 total, 8 up, 8 in 2024-04-22T15:44:26.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:26 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/3363684314' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/536309348"}]: dispatch 2024-04-22T15:44:26.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:26 smithi172 ceph-mon[65307]: from='client.54862 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:44:27.265 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch ps' 2024-04-22T15:44:27.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:27 smithi145 ceph-mon[88162]: from='client.54868 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:44:27.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:27 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/3363684314' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/536309348"}]': finished 2024-04-22T15:44:27.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:27 smithi145 ceph-mon[88162]: osdmap e161: 8 total, 8 up, 8 in 2024-04-22T15:44:27.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:27 smithi145 ceph-mon[82523]: from='client.54868 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:44:27.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:27 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/3363684314' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/536309348"}]': finished 2024-04-22T15:44:27.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:27 smithi145 ceph-mon[82523]: osdmap e161: 8 total, 8 up, 8 in 2024-04-22T15:44:28.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:27 smithi172 ceph-mon[65307]: from='client.54868 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:44:28.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:27 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/3363684314' entity='client.iscsi.foo.smithi145.ljpvqb' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.145:6800/536309348"}]': finished 2024-04-22T15:44:28.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:27 smithi172 ceph-mon[65307]: osdmap e161: 8 total, 8 up, 8 in 2024-04-22T15:44:28.927 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:28 smithi145 ceph-mon[88162]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:44:28.928 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:28 smithi145 ceph-mon[82523]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:44:29.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:28 smithi172 ceph-mon[65307]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail 2024-04-22T15:44:29.363 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:44:29.364 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (11m) 11s ago 33m - - 0.25.0 c8568f914cd2 1c58ccf37ffd 2024-04-22T15:44:29.364 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (10m) 70s ago 33m 76.5M - 9.4.12 2bacad6d85d8 2f825d15ab10 2024-04-22T15:44:29.364 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 running (16s) 11s ago 32m - - 3.7 3d983c26e278 67b9b9ffe63e 2024-04-22T15:44:29.364 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443,9283,8765 running (12m) 70s ago 38m 454M - 19.0.0-3223-g5883d494 3d983c26e278 e7500ef71604 2024-04-22T15:44:29.364 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (22m) 11s ago 39m - - 19.0.0-3223-g5883d494 3d983c26e278 b2de97566cf4 2024-04-22T15:44:29.364 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (8m) 11s ago 39m - 2048M 19.0.0-3223-g5883d494 3d983c26e278 7bfdc11120f4 2024-04-22T15:44:29.364 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (9m) 70s ago 38m 54.7M 2048M 19.0.0-3223-g5883d494 3d983c26e278 0a360b08a6d6 2024-04-22T15:44:29.364 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (8m) 11s ago 38m - 2048M 19.0.0-3223-g5883d494 3d983c26e278 38dcfa677c18 2024-04-22T15:44:29.364 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (12m) 11s ago 34m - - 1.5.0 0da6a335fe13 8c75c26fec05 2024-04-22T15:44:29.364 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (12m) 70s ago 34m 16.0M - 1.5.0 0da6a335fe13 8fd2949e2eaf 2024-04-22T15:44:29.364 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (6m) 11s ago 37m - 2992M 19.0.0-3223-g5883d494 3d983c26e278 e4bc06317af0 2024-04-22T15:44:29.364 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (6m) 11s ago 37m - 2992M 19.0.0-3223-g5883d494 3d983c26e278 d299f622f08d 2024-04-22T15:44:29.364 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (5m) 11s ago 37m - 2992M 19.0.0-3223-g5883d494 3d983c26e278 6aea4a926fe4 2024-04-22T15:44:29.365 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (4m) 11s ago 36m - 2992M 19.0.0-3223-g5883d494 3d983c26e278 c0cd2bdc1439 2024-04-22T15:44:29.365 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (3m) 70s ago 36m 51.4M 3248M 19.0.0-3223-g5883d494 3d983c26e278 7ac5776bc07a 2024-04-22T15:44:29.365 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (3m) 70s ago 35m 57.0M 3248M 19.0.0-3223-g5883d494 3d983c26e278 5f803668a17b 2024-04-22T15:44:29.365 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (2m) 70s ago 35m 48.0M 3248M 19.0.0-3223-g5883d494 3d983c26e278 9e049dd4678d 2024-04-22T15:44:29.365 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (2m) 70s ago 35m 47.2M 3248M 19.0.0-3223-g5883d494 3d983c26e278 951a8da56333 2024-04-22T15:44:29.365 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (11m) 70s ago 34m 77.0M - 2.43.0 a07b618ecd1d 51cf4df6e8d4 2024-04-22T15:44:29.365 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (80s) 11s ago 32m - - 19.0.0-3223-g5883d494 3d983c26e278 a4a0fbef80df 2024-04-22T15:44:29.365 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (74s) 70s ago 32m 73.8M - 19.0.0-3223-g5883d494 3d983c26e278 6f2627922357 2024-04-22T15:44:30.084 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:29 smithi145 ceph-mon[82523]: from='client.54874 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:44:30.085 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:29 smithi145 ceph-mon[88162]: from='client.54874 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:44:30.109 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph versions' 2024-04-22T15:44:30.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:29 smithi172 ceph-mon[65307]: from='client.54874 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:44:31.084 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:30 smithi145 ceph-mon[88162]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 856 B/s rd, 0 op/s 2024-04-22T15:44:31.084 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:30 smithi145 ceph-mon[82523]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 856 B/s rd, 0 op/s 2024-04-22T15:44:31.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:30 smithi172 ceph-mon[65307]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 856 B/s rd, 0 op/s 2024-04-22T15:44:32.096 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:44:32.097 INFO:teuthology.orchestra.run.smithi145.stdout: "mon": { 2024-04-22T15:44:32.097 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 3 2024-04-22T15:44:32.097 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:44:32.097 INFO:teuthology.orchestra.run.smithi145.stdout: "mgr": { 2024-04-22T15:44:32.097 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:44:32.097 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:44:32.097 INFO:teuthology.orchestra.run.smithi145.stdout: "osd": { 2024-04-22T15:44:32.097 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 8 2024-04-22T15:44:32.097 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:44:32.097 INFO:teuthology.orchestra.run.smithi145.stdout: "rgw": { 2024-04-22T15:44:32.097 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:44:32.097 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:44:32.097 INFO:teuthology.orchestra.run.smithi145.stdout: "overall": { 2024-04-22T15:44:32.097 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 15 2024-04-22T15:44:32.098 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:44:32.098 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:44:32.749 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:32 smithi145 ceph-mon[88162]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 734 B/s rd, 0 op/s 2024-04-22T15:44:32.749 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:32 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/2617850023' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:44:32.749 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:32 smithi145 ceph-mon[82523]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 734 B/s rd, 0 op/s 2024-04-22T15:44:32.749 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:32 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/2617850023' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:44:32.774 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-04-22T15:44:33.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:32 smithi172 ceph-mon[65307]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 734 B/s rd, 0 op/s 2024-04-22T15:44:33.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:32 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/2617850023' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:44:34.610 INFO:teuthology.orchestra.run.smithi145.stdout:wait for servicemap items w/ changing names to refresh 2024-04-22T15:44:35.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:34 smithi145 ceph-mon[82523]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:44:35.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:34 smithi145 ceph-mon[82523]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:44:35.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:44:35.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:34 smithi145 ceph-mon[88162]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:44:35.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:34 smithi145 ceph-mon[88162]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:44:35.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:44:35.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:34 smithi172 ceph-mon[65307]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:44:35.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:34 smithi172 ceph-mon[65307]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:44:35.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:44:35.372 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'sleep 60' 2024-04-22T15:44:35.622 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:44:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:44:35] "GET /metrics HTTP/1.1" 200 38494 "" "Prometheus/2.43.0" 2024-04-22T15:44:36.982 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:36 smithi145 ceph-mon[88162]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:44:36.983 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:36 smithi145 ceph-mon[82523]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:44:37.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:36 smithi172 ceph-mon[65307]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-22T15:44:39.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:38 smithi145 ceph-mon[82523]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 934 B/s rd, 0 op/s 2024-04-22T15:44:39.122 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:38 smithi145 ceph-mon[88162]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 934 B/s rd, 0 op/s 2024-04-22T15:44:39.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:38 smithi172 ceph-mon[65307]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 934 B/s rd, 0 op/s 2024-04-22T15:44:41.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:41 smithi145 ceph-mon[88162]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:44:41.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:41 smithi145 ceph-mon[82523]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:44:41.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:41 smithi172 ceph-mon[65307]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:44:43.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:43 smithi145 ceph-mon[88162]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:44:43.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:43 smithi145 ceph-mon[82523]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:44:43.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:43 smithi172 ceph-mon[65307]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:44:45.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:45 smithi145 ceph-mon[88162]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:44:45.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:45 smithi145 ceph-mon[88162]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:44:45.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:45 smithi145 ceph-mon[82523]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:44:45.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:45 smithi145 ceph-mon[82523]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:44:45.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:44:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:44:45] "GET /metrics HTTP/1.1" 200 38494 "" "Prometheus/2.43.0" 2024-04-22T15:44:45.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:45 smithi172 ceph-mon[65307]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:44:45.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:45 smithi172 ceph-mon[65307]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:44:47.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:47 smithi145 ceph-mon[82523]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:44:47.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:47 smithi145 ceph-mon[88162]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:44:47.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:47 smithi172 ceph-mon[65307]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:44:49.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:49 smithi145 ceph-mon[82523]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:44:49.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:49 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:44:49.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:49 smithi145 ceph-mon[88162]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:44:49.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:49 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:44:49.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:49 smithi172 ceph-mon[65307]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:44:49.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:49 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:44:51.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:51 smithi145 ceph-mon[82523]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:44:51.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:51 smithi145 ceph-mon[88162]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:44:51.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:51 smithi172 ceph-mon[65307]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:44:53.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:53 smithi145 ceph-mon[82523]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:44:53.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:53 smithi145 ceph-mon[88162]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:44:53.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:53 smithi172 ceph-mon[65307]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:44:55.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:55 smithi145 ceph-mon[82523]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:44:55.625 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:55 smithi145 ceph-mon[82523]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:44:55.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:55 smithi145 ceph-mon[88162]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:44:55.626 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:55 smithi145 ceph-mon[88162]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:44:55.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:44:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:44:55] "GET /metrics HTTP/1.1" 200 38497 "" "Prometheus/2.43.0" 2024-04-22T15:44:55.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:55 smithi172 ceph-mon[65307]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:44:55.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:55 smithi172 ceph-mon[65307]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:44:57.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:57 smithi145 ceph-mon[82523]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:44:57.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:57 smithi145 ceph-mon[88162]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:44:57.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:57 smithi172 ceph-mon[65307]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:44:59.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:44:59 smithi145 ceph-mon[82523]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:44:59.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:44:59 smithi145 ceph-mon[88162]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:44:59.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:44:59 smithi172 ceph-mon[65307]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:01.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:01 smithi172 ceph-mon[65307]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:01.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:01 smithi145 ceph-mon[82523]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:01.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:01 smithi145 ceph-mon[88162]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:03.648 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:03 smithi145 ceph-mon[88162]: pgmap v375: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:03.648 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:03 smithi145 ceph-mon[82523]: pgmap v375: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:03.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:03 smithi172 ceph-mon[65307]: pgmap v375: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:04.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:04 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:45:04.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:04 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:45:04.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:04 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:45:05.622 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:45:05 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:45:05] "GET /metrics HTTP/1.1" 200 38495 "" "Prometheus/2.43.0" 2024-04-22T15:45:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:05 smithi145 ceph-mon[88162]: pgmap v376: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:05.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:05 smithi145 ceph-mon[88162]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:45:05.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:05 smithi145 ceph-mon[82523]: pgmap v376: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:05.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:05 smithi145 ceph-mon[82523]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:45:05.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:05 smithi172 ceph-mon[65307]: pgmap v376: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:05.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:05 smithi172 ceph-mon[65307]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:45:07.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:07 smithi172 ceph-mon[65307]: pgmap v377: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:07.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:07 smithi145 ceph-mon[82523]: pgmap v377: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:07.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:07 smithi145 ceph-mon[88162]: pgmap v377: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:09.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:09 smithi172 ceph-mon[65307]: pgmap v378: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:09.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:09 smithi145 ceph-mon[82523]: pgmap v378: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:09.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:09 smithi145 ceph-mon[88162]: pgmap v378: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:11.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:11 smithi172 ceph-mon[65307]: pgmap v379: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:11.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:11 smithi145 ceph-mon[88162]: pgmap v379: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:11.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:11 smithi145 ceph-mon[82523]: pgmap v379: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:13.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:13 smithi172 ceph-mon[65307]: pgmap v380: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T15:45:13.806 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:13 smithi145 ceph-mon[82523]: pgmap v380: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T15:45:13.807 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:13 smithi145 ceph-mon[88162]: pgmap v380: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T15:45:15.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:15 smithi145 ceph-mon[82523]: pgmap v381: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:15.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:15 smithi145 ceph-mon[82523]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:45:15.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:15 smithi145 ceph-mon[88162]: pgmap v381: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:15.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:15 smithi145 ceph-mon[88162]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:45:15.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:45:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:45:15] "GET /metrics HTTP/1.1" 200 38495 "" "Prometheus/2.43.0" 2024-04-22T15:45:15.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:15 smithi172 ceph-mon[65307]: pgmap v381: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:15.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:15 smithi172 ceph-mon[65307]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:45:17.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:17 smithi172 ceph-mon[65307]: pgmap v382: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T15:45:17.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:17 smithi145 ceph-mon[88162]: pgmap v382: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T15:45:17.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:17 smithi145 ceph-mon[82523]: pgmap v382: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T15:45:18.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:18 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:45:18.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:18 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:45:18.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:18 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:45:18.718 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:18 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:45:18.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:18 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:45:18.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:18 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:45:18.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:18 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:45:18.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:18 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:45:18.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:18 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T15:45:18.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:18 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T15:45:18.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:18 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T15:45:18.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:18 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' 2024-04-22T15:45:19.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:19 smithi172 ceph-mon[65307]: pgmap v383: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T15:45:19.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:19 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:45:19.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:19 smithi145 ceph-mon[82523]: pgmap v383: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T15:45:19.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:19 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:45:19.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:19 smithi145 ceph-mon[88162]: pgmap v383: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T15:45:19.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:19 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:45:21.674 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:21 smithi145 ceph-mon[88162]: pgmap v384: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:21.675 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:21 smithi145 ceph-mon[82523]: pgmap v384: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:21.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:21 smithi172 ceph-mon[65307]: pgmap v384: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:23.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:23 smithi172 ceph-mon[65307]: pgmap v385: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:23.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:23 smithi145 ceph-mon[82523]: pgmap v385: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:23.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:23 smithi145 ceph-mon[88162]: pgmap v385: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:25.622 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:25 smithi145 ceph-mon[82523]: pgmap v386: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:25.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:25 smithi145 ceph-mon[82523]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:45:25.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:45:25 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:45:25] "GET /metrics HTTP/1.1" 200 38496 "" "Prometheus/2.43.0" 2024-04-22T15:45:25.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:25 smithi145 ceph-mon[88162]: pgmap v386: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:25.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:25 smithi145 ceph-mon[88162]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:45:25.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:25 smithi172 ceph-mon[65307]: pgmap v386: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:25.717 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:25 smithi172 ceph-mon[65307]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:45:26.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:26 smithi145 ceph-mon[88162]: pgmap v387: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T15:45:26.873 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:26 smithi145 ceph-mon[82523]: pgmap v387: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T15:45:26.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:26 smithi172 ceph-mon[65307]: pgmap v387: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T15:45:29.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:28 smithi145 ceph-mon[82523]: pgmap v388: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T15:45:29.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:28 smithi145 ceph-mon[88162]: pgmap v388: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T15:45:29.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:28 smithi172 ceph-mon[65307]: pgmap v388: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T15:45:31.122 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:30 smithi145 ceph-mon[88162]: pgmap v389: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:31.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:30 smithi145 ceph-mon[82523]: pgmap v389: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:31.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:30 smithi172 ceph-mon[65307]: pgmap v389: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:33.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:32 smithi145 ceph-mon[82523]: pgmap v390: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T15:45:33.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:32 smithi145 ceph-mon[88162]: pgmap v390: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T15:45:33.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:32 smithi172 ceph-mon[65307]: pgmap v390: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T15:45:35.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:34 smithi145 ceph-mon[82523]: pgmap v391: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:35.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:34 smithi145 ceph-mon[82523]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:45:35.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:34 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:45:35.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:34 smithi145 ceph-mon[88162]: pgmap v391: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:35.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:34 smithi145 ceph-mon[88162]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:45:35.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:34 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:45:35.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:34 smithi172 ceph-mon[65307]: pgmap v391: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:35.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:34 smithi172 ceph-mon[65307]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:45:35.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:34 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:45:35.622 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:45:35 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:45:35] "GET /metrics HTTP/1.1" 200 38495 "" "Prometheus/2.43.0" 2024-04-22T15:45:37.083 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:36 smithi145 ceph-mon[82523]: pgmap v392: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T15:45:37.083 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:36 smithi145 ceph-mon[88162]: pgmap v392: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T15:45:37.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:36 smithi172 ceph-mon[65307]: pgmap v392: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-22T15:45:37.862 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch ps' 2024-04-22T15:45:39.093 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:38 smithi145 ceph-mon[82523]: pgmap v393: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:39.093 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:38 smithi145 ceph-mon[88162]: pgmap v393: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:39.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:38 smithi172 ceph-mon[65307]: pgmap v393: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:39.846 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T15:45:39.846 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (12m) 82s ago 34m - - 0.25.0 c8568f914cd2 1c58ccf37ffd 2024-04-22T15:45:39.846 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi172 *:3000 running (12m) 2m ago 34m 76.5M - 9.4.12 2bacad6d85d8 2f825d15ab10 2024-04-22T15:45:39.846 INFO:teuthology.orchestra.run.smithi145.stdout:iscsi.foo.smithi145.ljpvqb smithi145 running (86s) 82s ago 33m - - 3.7 3d983c26e278 67b9b9ffe63e 2024-04-22T15:45:39.846 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi172 *:8443,9283,8765 running (13m) 2m ago 39m 454M - 19.0.0-3223-g5883d494 3d983c26e278 e7500ef71604 2024-04-22T15:45:39.847 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (23m) 82s ago 40m - - 19.0.0-3223-g5883d494 3d983c26e278 b2de97566cf4 2024-04-22T15:45:39.847 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (10m) 82s ago 41m - 2048M 19.0.0-3223-g5883d494 3d983c26e278 7bfdc11120f4 2024-04-22T15:45:39.847 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi172 running (10m) 2m ago 39m 54.7M 2048M 19.0.0-3223-g5883d494 3d983c26e278 0a360b08a6d6 2024-04-22T15:45:39.847 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (9m) 82s ago 39m - 2048M 19.0.0-3223-g5883d494 3d983c26e278 38dcfa677c18 2024-04-22T15:45:39.847 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (13m) 82s ago 35m - - 1.5.0 0da6a335fe13 8c75c26fec05 2024-04-22T15:45:39.847 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi172 *:9100 running (13m) 2m ago 35m 16.0M - 1.5.0 0da6a335fe13 8fd2949e2eaf 2024-04-22T15:45:39.847 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (7m) 82s ago 39m - 2992M 19.0.0-3223-g5883d494 3d983c26e278 e4bc06317af0 2024-04-22T15:45:39.847 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (7m) 82s ago 38m - 2992M 19.0.0-3223-g5883d494 3d983c26e278 d299f622f08d 2024-04-22T15:45:39.847 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (6m) 82s ago 38m - 2992M 19.0.0-3223-g5883d494 3d983c26e278 6aea4a926fe4 2024-04-22T15:45:39.847 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (5m) 82s ago 37m - 2992M 19.0.0-3223-g5883d494 3d983c26e278 c0cd2bdc1439 2024-04-22T15:45:39.847 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi172 running (5m) 2m ago 37m 51.4M 3248M 19.0.0-3223-g5883d494 3d983c26e278 7ac5776bc07a 2024-04-22T15:45:39.847 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi172 running (4m) 2m ago 37m 57.0M 3248M 19.0.0-3223-g5883d494 3d983c26e278 5f803668a17b 2024-04-22T15:45:39.847 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi172 running (3m) 2m ago 36m 48.0M 3248M 19.0.0-3223-g5883d494 3d983c26e278 9e049dd4678d 2024-04-22T15:45:39.847 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi172 running (3m) 2m ago 36m 47.2M 3248M 19.0.0-3223-g5883d494 3d983c26e278 951a8da56333 2024-04-22T15:45:39.847 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi172 *:9095 running (12m) 2m ago 35m 77.0M - 2.43.0 a07b618ecd1d 51cf4df6e8d4 2024-04-22T15:45:39.847 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi145.ygrqmi smithi145 *:8000 running (2m) 82s ago 34m - - 19.0.0-3223-g5883d494 3d983c26e278 a4a0fbef80df 2024-04-22T15:45:39.848 INFO:teuthology.orchestra.run.smithi145.stdout:rgw.foo.smithi172.glxwwa smithi172 *:8000 running (2m) 2m ago 33m 73.8M - 19.0.0-3223-g5883d494 3d983c26e278 6f2627922357 2024-04-22T15:45:40.556 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph versions' 2024-04-22T15:45:40.802 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:40 smithi145 ceph-mon[82523]: pgmap v394: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:40.802 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:40 smithi145 ceph-mon[82523]: from='client.54886 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:45:40.802 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:40 smithi145 ceph-mon[88162]: pgmap v394: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:40.802 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:40 smithi145 ceph-mon[88162]: from='client.54886 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:45:41.020 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:40 smithi172 ceph-mon[65307]: pgmap v394: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:41.020 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:40 smithi172 ceph-mon[65307]: from='client.54886 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:45:42.621 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:45:42.621 INFO:teuthology.orchestra.run.smithi145.stdout: "mon": { 2024-04-22T15:45:42.621 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 3 2024-04-22T15:45:42.621 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:45:42.621 INFO:teuthology.orchestra.run.smithi145.stdout: "mgr": { 2024-04-22T15:45:42.621 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:45:42.621 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:45:42.621 INFO:teuthology.orchestra.run.smithi145.stdout: "osd": { 2024-04-22T15:45:42.621 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 8 2024-04-22T15:45:42.621 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:45:42.621 INFO:teuthology.orchestra.run.smithi145.stdout: "rgw": { 2024-04-22T15:45:42.622 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 2 2024-04-22T15:45:42.622 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-04-22T15:45:42.622 INFO:teuthology.orchestra.run.smithi145.stdout: "overall": { 2024-04-22T15:45:42.622 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)": 15 2024-04-22T15:45:42.622 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-04-22T15:45:42.622 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:45:42.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:42 smithi145 ceph-mon[82523]: pgmap v395: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:42.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:42 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/1072911319' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:45:42.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:42 smithi145 ceph-mon[88162]: pgmap v395: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:42.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:42 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/1072911319' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:45:43.173 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:42 smithi172 ceph-mon[65307]: pgmap v395: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:43.173 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:42 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/1072911319' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:45:43.281 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch upgrade status' 2024-04-22T15:45:44.871 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:44 smithi145 ceph-mon[82523]: pgmap v396: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:44.871 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:44 smithi145 ceph-mon[82523]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:45:44.871 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:44 smithi145 ceph-mon[88162]: pgmap v396: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:44.871 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:44 smithi145 ceph-mon[88162]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:45:45.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:44 smithi172 ceph-mon[65307]: pgmap v396: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:45.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:44 smithi172 ceph-mon[65307]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:45:45.332 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:45:45.332 INFO:teuthology.orchestra.run.smithi145.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb", 2024-04-22T15:45:45.332 INFO:teuthology.orchestra.run.smithi145.stdout: "in_progress": true, 2024-04-22T15:45:45.333 INFO:teuthology.orchestra.run.smithi145.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-22T15:45:45.333 INFO:teuthology.orchestra.run.smithi145.stdout: "services_complete": [ 2024-04-22T15:45:45.333 INFO:teuthology.orchestra.run.smithi145.stdout: "iscsi", 2024-04-22T15:45:45.333 INFO:teuthology.orchestra.run.smithi145.stdout: "mgr", 2024-04-22T15:45:45.333 INFO:teuthology.orchestra.run.smithi145.stdout: "rgw", 2024-04-22T15:45:45.333 INFO:teuthology.orchestra.run.smithi145.stdout: "mon", 2024-04-22T15:45:45.333 INFO:teuthology.orchestra.run.smithi145.stdout: "osd" 2024-04-22T15:45:45.333 INFO:teuthology.orchestra.run.smithi145.stdout: ], 2024-04-22T15:45:45.333 INFO:teuthology.orchestra.run.smithi145.stdout: "progress": "16/21 daemons upgraded", 2024-04-22T15:45:45.333 INFO:teuthology.orchestra.run.smithi145.stdout: "message": "Error: UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi145.ljpvqb on host smithi145 failed.", 2024-04-22T15:45:45.333 INFO:teuthology.orchestra.run.smithi145.stdout: "is_paused": true 2024-04-22T15:45:45.333 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:45:45.622 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:45:45 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:45:45] "GET /metrics HTTP/1.1" 200 38495 "" "Prometheus/2.43.0" 2024-04-22T15:45:46.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:45 smithi145 ceph-mon[82523]: from='client.44967 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:45:46.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:45 smithi145 ceph-mon[88162]: from='client.44967 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:45:46.089 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph health detail' 2024-04-22T15:45:46.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:45 smithi172 ceph-mon[65307]: from='client.44967 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:45:47.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:46 smithi145 ceph-mon[82523]: pgmap v397: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:47.122 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:46 smithi145 ceph-mon[88162]: pgmap v397: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:47.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:46 smithi172 ceph-mon[65307]: pgmap v397: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:48.192 INFO:teuthology.orchestra.run.smithi145.stdout:HEALTH_WARN Upgrading daemon iscsi.foo.smithi145.ljpvqb on host smithi145 failed. 2024-04-22T15:45:48.192 INFO:teuthology.orchestra.run.smithi145.stdout:[WRN] UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi145.ljpvqb on host smithi145 failed. 2024-04-22T15:45:48.193 INFO:teuthology.orchestra.run.smithi145.stdout: Upgrade daemon: iscsi.foo.smithi145.ljpvqb: cephadm exited with an error code: 1, stderr: Redeploy daemon iscsi.foo.smithi145.ljpvqb ... 2024-04-22T15:45:48.193 INFO:teuthology.orchestra.run.smithi145.stdout:Creating ceph-iscsi config... 2024-04-22T15:45:48.193 INFO:teuthology.orchestra.run.smithi145.stdout:Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/iscsi-gateway.cfg 2024-04-22T15:45:48.193 INFO:teuthology.orchestra.run.smithi145.stdout:Write file: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/iscsi.foo.smithi145.ljpvqb/tcmu-runner-entrypoint.sh 2024-04-22T15:45:48.193 INFO:teuthology.orchestra.run.smithi145.stdout:Failed to trim old cgroups /sys/fs/cgroup/system.slice/system-ceph\x2d869afcfa\x2d00b9\x2d11ef\x2dbc93\x2dc7b262605968.slice/ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service 2024-04-22T15:45:48.193 INFO:teuthology.orchestra.run.smithi145.stdout:Non-zero exit code 1 from systemctl start ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb 2024-04-22T15:45:48.193 INFO:teuthology.orchestra.run.smithi145.stdout:systemctl: stderr Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:45:48.193 INFO:teuthology.orchestra.run.smithi145.stdout:systemctl: stderr See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:45:48.193 INFO:teuthology.orchestra.run.smithi145.stdout:Traceback (most recent call last): 2024-04-22T15:45:48.193 INFO:teuthology.orchestra.run.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-22T15:45:48.193 INFO:teuthology.orchestra.run.smithi145.stdout: return _run_code(code, main_globals, None, 2024-04-22T15:45:48.193 INFO:teuthology.orchestra.run.smithi145.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-22T15:45:48.193 INFO:teuthology.orchestra.run.smithi145.stdout: exec(code, run_globals) 2024-04-22T15:45:48.194 INFO:teuthology.orchestra.run.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5642, in 2024-04-22T15:45:48.194 INFO:teuthology.orchestra.run.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 5630, in main 2024-04-22T15:45:48.194 INFO:teuthology.orchestra.run.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3117, in command_deploy_from 2024-04-22T15:45:48.194 INFO:teuthology.orchestra.run.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3152, in _common_deploy 2024-04-22T15:45:48.194 INFO:teuthology.orchestra.run.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 3172, in _deploy_daemon_container 2024-04-22T15:45:48.194 INFO:teuthology.orchestra.run.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1087, in deploy_daemon 2024-04-22T15:45:48.194 INFO:teuthology.orchestra.run.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/__main__.py", line 1223, in deploy_daemon_units 2024-04-22T15:45:48.194 INFO:teuthology.orchestra.run.smithi145.stdout: File "/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/cephadm.6073b07ab069519e3286caa9b3ba4d63a482527cb6f564b205dd8e6cfafb5da0/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-22T15:45:48.194 INFO:teuthology.orchestra.run.smithi145.stdout:RuntimeError: Failed command: systemctl start ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb: Job for ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service failed because the control process exited with error code. 2024-04-22T15:45:48.194 INFO:teuthology.orchestra.run.smithi145.stdout:See "systemctl status ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" and "journalctl -xeu ceph-869afcfa-00b9-11ef-bc93-c7b262605968@iscsi.foo.smithi145.ljpvqb.service" for details. 2024-04-22T15:45:48.854 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:48 smithi145 ceph-mon[88162]: pgmap v398: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:48.855 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:48 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/198907926' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:45:48.855 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:48 smithi145 ceph-mon[82523]: pgmap v398: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:48.855 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:48 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/198907926' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:45:48.880 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-04-22T15:45:49.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:48 smithi172 ceph-mon[65307]: pgmap v398: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:49.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:48 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/198907926' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T15:45:50.081 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:49 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:45:50.081 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:49 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:45:50.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:49 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:45:50.917 INFO:teuthology.orchestra.run.smithi145.stdout:true 2024-04-22T15:45:51.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:50 smithi145 ceph-mon[82523]: pgmap v399: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:51.122 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:50 smithi145 ceph-mon[88162]: pgmap v399: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:51.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:50 smithi172 ceph-mon[65307]: pgmap v399: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:51.594 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-04-22T15:45:51.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:51 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/1836743812' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:45:51.873 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:51 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/1836743812' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:45:52.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:51 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/1836743812' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:45:52.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:52 smithi172 ceph-mon[65307]: pgmap v400: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:53.047 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:52 smithi145 ceph-mon[88162]: pgmap v400: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:53.047 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:52 smithi145 ceph-mon[82523]: pgmap v400: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:53.690 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-3223-g5883d494 (5883d494d3973b30b3f03e4a3f0e1e06e3c131aa) squid (dev)" 2024-04-22T15:45:54.036 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:53 smithi145 ceph-mon[88162]: from='client.? 172.21.15.145:0/2025915243' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:45:54.037 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:53 smithi145 ceph-mon[82523]: from='client.? 172.21.15.145:0/2025915243' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:45:54.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:53 smithi172 ceph-mon[65307]: from='client.? 172.21.15.145:0/2025915243' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T15:45:54.361 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -e sha1=5883d494d3973b30b3f03e4a3f0e1e06e3c131aa -- bash -c 'ceph orch ls | grep '"'"'^osd '"'"'' 2024-04-22T15:45:54.991 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:54 smithi145 ceph-mon[88162]: pgmap v401: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:54.991 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:54 smithi145 ceph-mon[88162]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:45:54.991 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:54 smithi145 ceph-mon[82523]: pgmap v401: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:54.991 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:54 smithi145 ceph-mon[82523]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:45:55.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:54 smithi172 ceph-mon[65307]: pgmap v401: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:45:55.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:54 smithi172 ceph-mon[65307]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:45:55.622 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:45:55 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:45:55] "GET /metrics HTTP/1.1" 200 38494 "" "Prometheus/2.43.0" 2024-04-22T15:45:56.396 INFO:teuthology.orchestra.run.smithi145.stdout:osd 8 2m ago - 2024-04-22T15:45:57.019 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-22T15:45:57.029 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi145.front.sepia.ceph.com 2024-04-22T15:45:57.030 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- bash -c 'ceph orch upgrade ls' 2024-04-22T15:45:57.054 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:56 smithi145 ceph-mon[88162]: pgmap v402: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:57.054 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:56 smithi145 ceph-mon[88162]: from='client.44991 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:45:57.054 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:56 smithi145 ceph-mon[82523]: pgmap v402: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:57.055 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:56 smithi145 ceph-mon[82523]: from='client.44991 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:45:57.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:56 smithi172 ceph-mon[65307]: pgmap v402: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:57.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:56 smithi172 ceph-mon[65307]: from='client.44991 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:45:58.872 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:58 smithi145 ceph-mon[88162]: pgmap v403: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:58.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:58 smithi145 ceph-mon[82523]: pgmap v403: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:45:59.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:58 smithi172 ceph-mon[65307]: pgmap v403: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:46:00.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:45:59 smithi145 ceph-mon[82523]: from='client.44997 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:46:00.122 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:45:59 smithi145 ceph-mon[88162]: from='client.44997 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:46:00.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:45:59 smithi172 ceph-mon[65307]: from='client.44997 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:46:00.526 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-04-22T15:46:00.527 INFO:teuthology.orchestra.run.smithi145.stdout: "image": "quay.io/ceph/ceph", 2024-04-22T15:46:00.527 INFO:teuthology.orchestra.run.smithi145.stdout: "registry": "quay.io", 2024-04-22T15:46:00.527 INFO:teuthology.orchestra.run.smithi145.stdout: "bare_image": "ceph/ceph", 2024-04-22T15:46:00.527 INFO:teuthology.orchestra.run.smithi145.stdout: "versions": [] 2024-04-22T15:46:00.527 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-04-22T15:46:01.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:46:00 smithi145 ceph-mon[82523]: pgmap v404: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:46:01.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:46:00 smithi145 ceph-mon[88162]: pgmap v404: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:46:01.155 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0' 2024-04-22T15:46:01.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:46:00 smithi172 ceph-mon[65307]: pgmap v404: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:46:03.122 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:46:02 smithi145 ceph-mon[88162]: pgmap v405: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:46:03.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:46:02 smithi145 ceph-mon[82523]: pgmap v405: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:46:03.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:46:02 smithi172 ceph-mon[65307]: pgmap v405: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:46:04.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:46:03 smithi145 ceph-mon[82523]: from='client.54934 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:46:04.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:46:03 smithi145 ceph-mon[88162]: from='client.54934 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:46:04.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:46:03 smithi172 ceph-mon[65307]: from='client.54934 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:46:04.287 INFO:teuthology.orchestra.run.smithi145.stdout: "16.2.0", 2024-04-22T15:46:04.865 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:46:04 smithi145 ceph-mon[88162]: pgmap v406: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:46:04.866 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:46:04 smithi145 ceph-mon[88162]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:46:04.866 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:46:04 smithi145 ceph-mon[88162]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:46:04.866 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:46:04 smithi145 ceph-mon[82523]: pgmap v406: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:46:04.866 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:46:04 smithi145 ceph-mon[82523]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:46:04.866 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:46:04 smithi145 ceph-mon[82523]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:46:04.893 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2' 2024-04-22T15:46:05.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:46:04 smithi172 ceph-mon[65307]: pgmap v406: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T15:46:05.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:46:04 smithi172 ceph-mon[65307]: from='client.44844 -' entity='client.iscsi.foo.smithi145.ljpvqb' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T15:46:05.218 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:46:04 smithi172 ceph-mon[65307]: from='mgr.44106 172.21.15.145:0/2141701593' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-22T15:46:05.536 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:46:05 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: ::ffff:172.21.15.172 - - [22/Apr/2024:15:46:05] "GET /metrics HTTP/1.1" 200 38494 "" "Prometheus/2.43.0" 2024-04-22T15:46:07.122 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:46:06 smithi145 ceph-mon[82523]: pgmap v407: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:46:07.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:46:06 smithi145 ceph-mon[88162]: pgmap v407: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:46:07.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:46:06 smithi172 ceph-mon[65307]: pgmap v407: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:46:08.125 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:46:07 smithi145 ceph-mon[82523]: from='client.45009 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:46:08.125 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:46:07 smithi145 ceph-mon[88162]: from='client.45009 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:46:08.151 INFO:teuthology.orchestra.run.smithi145.stdout: "v16.2.2", 2024-04-22T15:46:08.151 INFO:teuthology.orchestra.run.smithi145.stdout: "v16.2.2-20210505", 2024-04-22T15:46:08.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:46:07 smithi172 ceph-mon[65307]: from='client.45009 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T15:46:08.760 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-22T15:46:08.769 INFO:tasks.cephadm:Teardown begin 2024-04-22T15:46:08.769 DEBUG:teuthology.orchestra.run.smithi145:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-22T15:46:08.798 DEBUG:teuthology.orchestra.run.smithi172:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-22T15:46:08.831 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-22T15:46:08.832 DEBUG:teuthology.orchestra.run.smithi145:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-22T15:46:08.856 DEBUG:teuthology.orchestra.run.smithi172:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-22T15:46:08.888 INFO:tasks.cephadm:Stopping all daemons... 2024-04-22T15:46:08.889 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-04-22T15:46:08.889 DEBUG:teuthology.orchestra.run.smithi145:> sudo systemctl stop ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mon.a 2024-04-22T15:46:09.122 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:46:08 smithi145 ceph-mon[88162]: pgmap v408: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:46:09.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:46:08 smithi145 ceph-mon[82523]: pgmap v408: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:46:09.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:46:08 smithi145 systemd[1]: Stopping Ceph mon.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:46:09.217 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:46:08 smithi172 ceph-mon[65307]: pgmap v408: 161 pgs: 161 active+clean; 585 KiB data, 254 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T15:46:09.552 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:46:09 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-a[82519]: 2024-04-22T15:46:09.207+0000 7f0781488700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:46:09.552 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:46:09 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-a[82519]: 2024-04-22T15:46:09.207+0000 7f0781488700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-04-22T15:46:09.552 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:46:09 smithi145 podman[134237]: 2024-04-22 15:46:09.279355803 +0000 UTC m=+0.201337723 container died 7bfdc11120f4affc98db5c05200905b159038265bb472095719837905fe1fb72 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-a, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-04-22T15:46:09.553 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:46:09 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:46:09] ENGINE Bus STOPPING 2024-04-22T15:46:09.553 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:46:09 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:46:09] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-22T15:46:09.553 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:46:09 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:46:09] ENGINE Bus STOPPED 2024-04-22T15:46:09.553 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:46:09 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:46:09] ENGINE Bus STARTING 2024-04-22T15:46:09.553 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:46:09 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: CherryPy Checker: 2024-04-22T15:46:09.553 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:46:09 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: The Application mounted at '' has an empty config. 2024-04-22T15:46:09.553 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:46:09 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:46:09.553 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:46:09 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:46:09] ENGINE Serving on http://:::9283 2024-04-22T15:46:09.553 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:46:09 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:46:09] ENGINE Bus STARTED 2024-04-22T15:46:09.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:46:09 smithi145 podman[134237]: 2024-04-22 15:46:09.641285403 +0000 UTC m=+0.563267330 container cleanup 7bfdc11120f4affc98db5c05200905b159038265bb472095719837905fe1fb72 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-a, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.vendor=CentOS) 2024-04-22T15:46:09.872 INFO:journalctl@ceph.mon.a.smithi145.stdout:Apr 22 15:46:09 smithi145 bash[134237]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-a 2024-04-22T15:46:10.001 DEBUG:teuthology.orchestra.run.smithi145:> sudo pkill -f 'journalctl -f -n 0 -u ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mon.a.service' 2024-04-22T15:46:10.051 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T15:46:10.051 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-04-22T15:46:10.052 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-04-22T15:46:10.052 DEBUG:teuthology.orchestra.run.smithi145:> sudo systemctl stop ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mon.c 2024-04-22T15:46:10.365 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:46:10 smithi145 systemd[1]: Stopping Ceph mon.c for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:46:10.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:46:10 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-c[88158]: 2024-04-22T15:46:10.361+0000 7f41eeefb700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:46:10.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:46:10 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-c[88158]: 2024-04-22T15:46:10.361+0000 7f41eeefb700 -1 mon.c@1(peon) e4 *** Got Signal Terminated *** 2024-04-22T15:46:10.622 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:46:10 smithi145 podman[134370]: 2024-04-22 15:46:10.383477132 +0000 UTC m=+0.155984016 container died 38dcfa677c181defaaf0e05ccaa8446ede97daeedabb7150cb2dec4bddda3ff2 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-c, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T15:46:10.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:46:10 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:46:10] ENGINE Bus STOPPING 2024-04-22T15:46:10.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:46:10 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:46:10] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-22T15:46:10.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:46:10 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:46:10] ENGINE Bus STOPPED 2024-04-22T15:46:10.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:46:10 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:46:10] ENGINE Bus STARTING 2024-04-22T15:46:10.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:46:10 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:46:10] ENGINE Serving on http://:::9283 2024-04-22T15:46:10.623 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:46:10 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: [22/Apr/2024:15:46:10] ENGINE Bus STARTED 2024-04-22T15:46:11.041 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:46:10 smithi145 podman[134370]: 2024-04-22 15:46:10.804503741 +0000 UTC m=+0.577010613 container cleanup 38dcfa677c181defaaf0e05ccaa8446ede97daeedabb7150cb2dec4bddda3ff2 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-c, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T15:46:11.041 INFO:journalctl@ceph.mon.c.smithi145.stdout:Apr 22 15:46:10 smithi145 bash[134370]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-c 2024-04-22T15:46:11.143 DEBUG:teuthology.orchestra.run.smithi145:> sudo pkill -f 'journalctl -f -n 0 -u ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mon.c.service' 2024-04-22T15:46:11.176 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T15:46:11.177 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-04-22T15:46:11.177 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-04-22T15:46:11.177 DEBUG:teuthology.orchestra.run.smithi172:> sudo systemctl stop ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mon.b 2024-04-22T15:46:11.468 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:46:11 smithi172 systemd[1]: Stopping Ceph mon.b for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:46:11.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:46:11 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-b[65303]: 2024-04-22T15:46:11.517+0000 7fd1ebfba700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:46:11.967 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:46:11 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-b[65303]: 2024-04-22T15:46:11.517+0000 7fd1ebfba700 -1 mon.b@2(peon) e4 *** Got Signal Terminated *** 2024-04-22T15:46:11.968 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:46:11 smithi172 podman[99911]: 2024-04-22 15:46:11.711772119 +0000 UTC m=+0.336825442 container died 0a360b08a6d618e8724ec24b720135e49374ef5b5ec35dab2de8a8f4c2e8c15e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-b, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:46:12.330 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:46:12 smithi172 podman[99911]: 2024-04-22 15:46:12.134209695 +0000 UTC m=+0.759263027 container cleanup 0a360b08a6d618e8724ec24b720135e49374ef5b5ec35dab2de8a8f4c2e8c15e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-b, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:46:12.330 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:46:12 smithi172 bash[99911]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-b 2024-04-22T15:46:12.566 DEBUG:teuthology.orchestra.run.smithi172:> sudo pkill -f 'journalctl -f -n 0 -u ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mon.b.service' 2024-04-22T15:46:12.587 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:46:12 smithi172 podman[99925]: 2024-04-22 15:46:12.327704918 +0000 UTC m=+0.612998541 container remove 0a360b08a6d618e8724ec24b720135e49374ef5b5ec35dab2de8a8f4c2e8c15e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mon-b, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416) 2024-04-22T15:46:12.587 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:46:12 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mon.b.service: Deactivated successfully. 2024-04-22T15:46:12.587 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:46:12 smithi172 systemd[1]: Stopped Ceph mon.b for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:46:12.587 INFO:journalctl@ceph.mon.b.smithi172.stdout:Apr 22 15:46:12 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mon.b.service: Consumed 7.908s CPU time. 2024-04-22T15:46:12.613 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T15:46:12.614 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-04-22T15:46:12.614 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-04-22T15:46:12.614 DEBUG:teuthology.orchestra.run.smithi145:> sudo systemctl stop ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mgr.y 2024-04-22T15:46:12.928 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:46:12 smithi145 systemd[1]: Stopping Ceph mgr.y for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:46:13.237 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:46:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:46:12.924+0000 7f181fa30700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:46:13.237 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:46:12 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y[60905]: 2024-04-22T15:46:12.924+0000 7f181fa30700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-04-22T15:46:13.237 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:46:12 smithi145 podman[134501]: 2024-04-22 15:46:12.96627149 +0000 UTC m=+0.167369777 container died b2de97566cf478b98326f7375fb502753c3be42e46892d4d1d3d363935886d49 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-04-22T15:46:13.614 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:46:13 smithi145 podman[134501]: 2024-04-22 15:46:13.344050628 +0000 UTC m=+0.545148918 container cleanup b2de97566cf478b98326f7375fb502753c3be42e46892d4d1d3d363935886d49 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T15:46:13.614 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Apr 22 15:46:13 smithi145 bash[134501]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-y 2024-04-22T15:46:13.776 DEBUG:teuthology.orchestra.run.smithi145:> sudo pkill -f 'journalctl -f -n 0 -u ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mgr.y.service' 2024-04-22T15:46:13.810 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T15:46:13.811 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-04-22T15:46:13.811 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-04-22T15:46:13.811 DEBUG:teuthology.orchestra.run.smithi172:> sudo systemctl stop ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mgr.x 2024-04-22T15:46:14.180 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:46:13 smithi172 systemd[1]: Stopping Ceph mgr.x for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:46:14.469 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:46:14 smithi172 podman[100031]: 2024-04-22 15:46:14.178482808 +0000 UTC m=+0.165010091 container died e7500ef71604b12a58c6d95ac7ea1e4da259e385e11bc16ba0f6f0f44339ac00 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, maintainer=Guillaume Abrioux ) 2024-04-22T15:46:14.912 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:46:14 smithi172 podman[100031]: 2024-04-22 15:46:14.505575102 +0000 UTC m=+0.492102387 container cleanup e7500ef71604b12a58c6d95ac7ea1e4da259e385e11bc16ba0f6f0f44339ac00 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, org.label-schema.build-date=20240416) 2024-04-22T15:46:14.912 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:46:14 smithi172 bash[100031]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x 2024-04-22T15:46:14.912 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:46:14 smithi172 podman[100045]: 2024-04-22 15:46:14.690751274 +0000 UTC m=+0.513713094 container remove e7500ef71604b12a58c6d95ac7ea1e4da259e385e11bc16ba0f6f0f44339ac00 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-mgr-x, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True) 2024-04-22T15:46:14.912 INFO:journalctl@ceph.mgr.x.smithi172.stdout:Apr 22 15:46:14 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-22T15:46:14.934 DEBUG:teuthology.orchestra.run.smithi172:> sudo pkill -f 'journalctl -f -n 0 -u ceph-869afcfa-00b9-11ef-bc93-c7b262605968@mgr.x.service' 2024-04-22T15:46:14.968 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T15:46:14.968 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-04-22T15:46:14.968 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-04-22T15:46:14.968 DEBUG:teuthology.orchestra.run.smithi145:> sudo systemctl stop ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.0 2024-04-22T15:46:15.267 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:46:15 smithi145 systemd[1]: Stopping Ceph osd.0 for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:46:15.622 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:46:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0[96579]: 2024-04-22T15:46:15.263+0000 7f69c4728700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:46:15.622 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:46:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0[96579]: 2024-04-22T15:46:15.263+0000 7f69c4728700 -1 osd.0 161 *** Got signal Terminated *** 2024-04-22T15:46:15.622 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:46:15 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0[96579]: 2024-04-22T15:46:15.263+0000 7f69c4728700 -1 osd.0 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T15:46:20.918 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:46:20 smithi145 podman[134622]: 2024-04-22 15:46:20.622721281 +0000 UTC m=+5.463827592 container died e4bc06317af0791486085859755b3e54ee589a1a6a1211217bb81e9dd060aa6e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-04-22T15:46:21.360 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:46:21 smithi145 podman[134622]: 2024-04-22 15:46:21.098084265 +0000 UTC m=+5.939190568 container cleanup e4bc06317af0791486085859755b3e54ee589a1a6a1211217bb81e9dd060aa6e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-04-22T15:46:21.360 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:46:21 smithi145 bash[134622]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0 2024-04-22T15:46:21.622 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:46:21 smithi145 podman[134639]: 2024-04-22 15:46:21.357906081 +0000 UTC m=+0.730079094 container remove e4bc06317af0791486085859755b3e54ee589a1a6a1211217bb81e9dd060aa6e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T15:46:21.956 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:46:21 smithi145 podman[134734]: 2024-04-22 15:46:21.558452037 +0000 UTC m=+0.017981888 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:46:21.956 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:46:21 smithi145 podman[134734]: 2024-04-22 15:46:21.701190885 +0000 UTC m=+0.160720718 container create 5f49a4b56c687c0b2c5ea7192ceb0001e194ee18e28cc88dc4ee27aefce914dd (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0-deactivate, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-04-22T15:46:21.956 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:46:21 smithi145 podman[134734]: 2024-04-22 15:46:21.953901563 +0000 UTC m=+0.413431395 container init 5f49a4b56c687c0b2c5ea7192ceb0001e194ee18e28cc88dc4ee27aefce914dd (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0-deactivate, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, CEPH_POINT_RELEASE=) 2024-04-22T15:46:22.259 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:46:21 smithi145 podman[134734]: 2024-04-22 15:46:21.957318233 +0000 UTC m=+0.416848071 container start 5f49a4b56c687c0b2c5ea7192ceb0001e194ee18e28cc88dc4ee27aefce914dd (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:46:22.259 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:46:22 smithi145 podman[134734]: 2024-04-22 15:46:22.062164488 +0000 UTC m=+0.521694325 container attach 5f49a4b56c687c0b2c5ea7192ceb0001e194ee18e28cc88dc4ee27aefce914dd (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0-deactivate, ceph=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-04-22T15:46:22.551 INFO:journalctl@ceph.osd.0.smithi145.stdout:Apr 22 15:46:22 smithi145 podman[134734]: 2024-04-22 15:46:22.257505651 +0000 UTC m=+0.717035491 container died 5f49a4b56c687c0b2c5ea7192ceb0001e194ee18e28cc88dc4ee27aefce914dd (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-04-22T15:46:22.859 DEBUG:teuthology.orchestra.run.smithi145:> sudo pkill -f 'journalctl -f -n 0 -u ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.0.service' 2024-04-22T15:46:22.897 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T15:46:22.897 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-04-22T15:46:22.897 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-04-22T15:46:22.897 DEBUG:teuthology.orchestra.run.smithi145:> sudo systemctl stop ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.1 2024-04-22T15:46:23.122 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:46:22 smithi145 systemd[1]: Stopping Ceph osd.1 for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:46:23.622 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:46:23 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1[101486]: 2024-04-22T15:46:23.270+0000 7f44f97e4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:46:23.622 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:46:23 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1[101486]: 2024-04-22T15:46:23.270+0000 7f44f97e4700 -1 osd.1 161 *** Got signal Terminated *** 2024-04-22T15:46:23.622 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:46:23 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1[101486]: 2024-04-22T15:46:23.270+0000 7f44f97e4700 -1 osd.1 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T15:46:29.014 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:46:28 smithi145 podman[134850]: 2024-04-22 15:46:28.710709071 +0000 UTC m=+5.555130162 container died d299f622f08dc5f13aa1cd57d8097cc8d408a8bb8fe5a65adde86ba21599a304 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:46:29.372 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:46:29 smithi145 podman[134850]: 2024-04-22 15:46:29.126785537 +0000 UTC m=+5.971206629 container cleanup d299f622f08dc5f13aa1cd57d8097cc8d408a8bb8fe5a65adde86ba21599a304 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True) 2024-04-22T15:46:29.372 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:46:29 smithi145 bash[134850]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1 2024-04-22T15:46:29.622 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:46:29 smithi145 podman[134869]: 2024-04-22 15:46:29.369816765 +0000 UTC m=+0.657137670 container remove d299f622f08dc5f13aa1cd57d8097cc8d408a8bb8fe5a65adde86ba21599a304 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, ceph=True) 2024-04-22T15:46:30.108 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:46:29 smithi145 podman[134963]: 2024-04-22 15:46:29.704906891 +0000 UTC m=+0.018255096 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:46:30.109 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:46:29 smithi145 podman[134963]: 2024-04-22 15:46:29.803895314 +0000 UTC m=+0.117243505 container create a56a12337840d183951c1d6d937df48b6ef446ccfe369ecd782ef365391f631e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.build-date=20240416, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True) 2024-04-22T15:46:30.109 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:46:30 smithi145 podman[134963]: 2024-04-22 15:46:30.023100011 +0000 UTC m=+0.336448200 container init a56a12337840d183951c1d6d937df48b6ef446ccfe369ecd782ef365391f631e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416) 2024-04-22T15:46:30.109 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:46:30 smithi145 podman[134963]: 2024-04-22 15:46:30.026204762 +0000 UTC m=+0.339552960 container start a56a12337840d183951c1d6d937df48b6ef446ccfe369ecd782ef365391f631e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-04-22T15:46:30.109 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:46:30 smithi145 podman[134963]: 2024-04-22 15:46:30.106393402 +0000 UTC m=+0.419741600 container attach a56a12337840d183951c1d6d937df48b6ef446ccfe369ecd782ef365391f631e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240416, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-22T15:46:30.373 INFO:journalctl@ceph.osd.1.smithi145.stdout:Apr 22 15:46:30 smithi145 podman[134963]: 2024-04-22 15:46:30.315896047 +0000 UTC m=+0.629244241 container died a56a12337840d183951c1d6d937df48b6ef446ccfe369ecd782ef365391f631e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T15:46:30.888 DEBUG:teuthology.orchestra.run.smithi145:> sudo pkill -f 'journalctl -f -n 0 -u ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.1.service' 2024-04-22T15:46:30.926 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T15:46:30.926 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-04-22T15:46:30.926 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-04-22T15:46:30.927 DEBUG:teuthology.orchestra.run.smithi145:> sudo systemctl stop ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.2 2024-04-22T15:46:31.122 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:46:31 smithi145 systemd[1]: Stopping Ceph osd.2 for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:46:31.622 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:46:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2[108325]: 2024-04-22T15:46:31.245+0000 7f7eeb044700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:46:31.622 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:46:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2[108325]: 2024-04-22T15:46:31.245+0000 7f7eeb044700 -1 osd.2 161 *** Got signal Terminated *** 2024-04-22T15:46:31.623 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:46:31 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2[108325]: 2024-04-22T15:46:31.245+0000 7f7eeb044700 -1 osd.2 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T15:46:37.031 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:46:36 smithi145 podman[135069]: 2024-04-22 15:46:36.717040079 +0000 UTC m=+5.568538900 container died 6aea4a926fe471881ad9b87309f4bae1ff43c8c78ad22cd1591b8ec46e6b0b1b (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-04-22T15:46:37.031 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:46:37 smithi145 podman[135069]: 2024-04-22 15:46:37.029167004 +0000 UTC m=+5.880665822 container cleanup 6aea4a926fe471881ad9b87309f4bae1ff43c8c78ad22cd1591b8ec46e6b0b1b (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-04-22T15:46:37.372 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:46:37 smithi145 bash[135069]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2 2024-04-22T15:46:37.372 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:46:37 smithi145 podman[135086]: 2024-04-22 15:46:37.272125386 +0000 UTC m=+0.556054610 container remove 6aea4a926fe471881ad9b87309f4bae1ff43c8c78ad22cd1591b8ec46e6b0b1b (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:46:37.872 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:46:37 smithi145 podman[135179]: 2024-04-22 15:46:37.47757362 +0000 UTC m=+0.018366142 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:46:37.872 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:46:37 smithi145 podman[135179]: 2024-04-22 15:46:37.589472271 +0000 UTC m=+0.130264781 container create 02c9fc47a6e8da02e49b6ce4738e1cd32d8003fcca1c42196c5d1e1645ed087e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2-deactivate, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-04-22T15:46:37.872 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:46:37 smithi145 podman[135179]: 2024-04-22 15:46:37.808844325 +0000 UTC m=+0.349636839 container init 02c9fc47a6e8da02e49b6ce4738e1cd32d8003fcca1c42196c5d1e1645ed087e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416) 2024-04-22T15:46:37.873 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:46:37 smithi145 podman[135179]: 2024-04-22 15:46:37.812302524 +0000 UTC m=+0.353095035 container start 02c9fc47a6e8da02e49b6ce4738e1cd32d8003fcca1c42196c5d1e1645ed087e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2-deactivate, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-04-22T15:46:38.372 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:46:37 smithi145 podman[135179]: 2024-04-22 15:46:37.896883404 +0000 UTC m=+0.437675939 container attach 02c9fc47a6e8da02e49b6ce4738e1cd32d8003fcca1c42196c5d1e1645ed087e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-04-22T15:46:38.373 INFO:journalctl@ceph.osd.2.smithi145.stdout:Apr 22 15:46:38 smithi145 podman[135179]: 2024-04-22 15:46:38.115231333 +0000 UTC m=+0.656023840 container died 02c9fc47a6e8da02e49b6ce4738e1cd32d8003fcca1c42196c5d1e1645ed087e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-04-22T15:46:38.740 DEBUG:teuthology.orchestra.run.smithi145:> sudo pkill -f 'journalctl -f -n 0 -u ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.2.service' 2024-04-22T15:46:38.777 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T15:46:38.778 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-04-22T15:46:38.778 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-22T15:46:38.778 DEBUG:teuthology.orchestra.run.smithi145:> sudo systemctl stop ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.3 2024-04-22T15:46:39.101 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:46:38 smithi145 systemd[1]: Stopping Ceph osd.3 for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:46:39.372 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:46:39 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3[115455]: 2024-04-22T15:46:39.097+0000 7fe7771b0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:46:39.372 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:46:39 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3[115455]: 2024-04-22T15:46:39.097+0000 7fe7771b0700 -1 osd.3 161 *** Got signal Terminated *** 2024-04-22T15:46:39.372 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:46:39 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3[115455]: 2024-04-22T15:46:39.097+0000 7fe7771b0700 -1 osd.3 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T15:46:41.622 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:46:41 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3[115455]: 2024-04-22T15:46:41.202+0000 7fe7737a4700 -1 osd.3 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:16.535380+0000 front 2024-04-22T15:46:16.535337+0000 (oldest deadline 2024-04-22T15:46:40.635454+0000) 2024-04-22T15:46:42.217 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:46:41 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:46:41.871+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:46:42.622 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:46:42 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3[115455]: 2024-04-22T15:46:42.240+0000 7fe7737a4700 -1 osd.3 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:16.535380+0000 front 2024-04-22T15:46:16.535337+0000 (oldest deadline 2024-04-22T15:46:40.635454+0000) 2024-04-22T15:46:42.635 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:46:42 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:46:42.396+0000 7fc58549f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:17.214788+0000 front 2024-04-22T15:46:17.214725+0000 (oldest deadline 2024-04-22T15:46:41.914579+0000) 2024-04-22T15:46:43.217 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:46:42 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:46:42.912+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:46:43.560 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:46:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3[115455]: 2024-04-22T15:46:43.248+0000 7fe7737a4700 -1 osd.3 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:16.535380+0000 front 2024-04-22T15:46:16.535337+0000 (oldest deadline 2024-04-22T15:46:40.635454+0000) 2024-04-22T15:46:43.717 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:46:43 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4[76239]: 2024-04-22T15:46:43.264+0000 7ff5b6a73700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.158272+0000 front 2024-04-22T15:46:18.158290+0000 (oldest deadline 2024-04-22T15:46:42.858188+0000) 2024-04-22T15:46:43.717 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:46:43 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:46:43.434+0000 7fc58549f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:17.214788+0000 front 2024-04-22T15:46:17.214725+0000 (oldest deadline 2024-04-22T15:46:41.914579+0000) 2024-04-22T15:46:43.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:46:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[75967]: ts=2024-04-22T15:46:43.558Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi145:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-04-22T15:46:43.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:46:43 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[75967]: ts=2024-04-22T15:46:43.558Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi172.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.172:8443: connect: connection refused" 2024-04-22T15:46:44.217 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:46:43 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:46:43.877+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:46:44.717 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:46:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4[76239]: 2024-04-22T15:46:44.225+0000 7ff5b6a73700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.158272+0000 front 2024-04-22T15:46:18.158290+0000 (oldest deadline 2024-04-22T15:46:42.858188+0000) 2024-04-22T15:46:44.718 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:46:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:46:44.470+0000 7fc58549f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:17.214788+0000 front 2024-04-22T15:46:17.214725+0000 (oldest deadline 2024-04-22T15:46:41.914579+0000) 2024-04-22T15:46:44.908 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:46:44 smithi145 podman[135285]: 2024-04-22 15:46:44.630952122 +0000 UTC m=+5.638800981 container died c0cd2bdc143986b2cc06e2c160448939217a663c4b9b40e38b7e779f9b379b63 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-04-22T15:46:45.217 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:46:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:46:44.836+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:46:45.218 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:46:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:46:44.993Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.145:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.145:8765: connect: connection refused" 2024-04-22T15:46:45.218 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:46:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:46:44.994Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.145:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.145:8765: connect: connection refused" 2024-04-22T15:46:45.218 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:46:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:46:44.995Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.145:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.145:8765: connect: connection refused" 2024-04-22T15:46:45.218 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:46:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:46:44.995Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.145:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.145:8765: connect: connection refused" 2024-04-22T15:46:45.218 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:46:44 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:46:44.995Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.145:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.145:8765: connect: connection refused" 2024-04-22T15:46:45.250 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:46:44 smithi145 podman[135285]: 2024-04-22 15:46:44.997426235 +0000 UTC m=+6.005275105 container cleanup c0cd2bdc143986b2cc06e2c160448939217a663c4b9b40e38b7e779f9b379b63 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, io.buildah.version=1.33.5) 2024-04-22T15:46:45.251 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:46:44 smithi145 bash[135285]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3 2024-04-22T15:46:45.601 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:46:45 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4[76239]: 2024-04-22T15:46:45.251+0000 7ff5b6a73700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.158272+0000 front 2024-04-22T15:46:18.158290+0000 (oldest deadline 2024-04-22T15:46:42.858188+0000) 2024-04-22T15:46:45.602 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:46:45 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:46:45.443+0000 7fc58549f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:17.214788+0000 front 2024-04-22T15:46:17.214725+0000 (oldest deadline 2024-04-22T15:46:41.914579+0000) 2024-04-22T15:46:45.612 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:46:45 smithi145 podman[135302]: 2024-04-22 15:46:45.248789793 +0000 UTC m=+0.615131907 container remove c0cd2bdc143986b2cc06e2c160448939217a663c4b9b40e38b7e779f9b379b63 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-04-22T15:46:45.872 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:46:45 smithi145 podman[135391]: 2024-04-22 15:46:45.513889519 +0000 UTC m=+0.018309235 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:46:45.873 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:46:45 smithi145 podman[135391]: 2024-04-22 15:46:45.615921807 +0000 UTC m=+0.120341509 container create 29e635e6fe926d0b481e1122cd4cd7c5e7a45472b11d1e75334cde1ed640df31 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, maintainer=Guillaume Abrioux ) 2024-04-22T15:46:45.873 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:46:45 smithi145 podman[135391]: 2024-04-22 15:46:45.801934375 +0000 UTC m=+0.306354082 container init 29e635e6fe926d0b481e1122cd4cd7c5e7a45472b11d1e75334cde1ed640df31 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416) 2024-04-22T15:46:45.873 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:46:45 smithi145 podman[135391]: 2024-04-22 15:46:45.804910345 +0000 UTC m=+0.309330052 container start 29e635e6fe926d0b481e1122cd4cd7c5e7a45472b11d1e75334cde1ed640df31 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-22T15:46:45.967 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:46:45 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:46:45.599+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:19.680121+0000 front 2024-04-22T15:46:19.680173+0000 (oldest deadline 2024-04-22T15:46:44.979938+0000) 2024-04-22T15:46:45.968 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:46:45 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:46:45.829+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:46:46.372 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:46:45 smithi145 podman[135391]: 2024-04-22 15:46:45.884991062 +0000 UTC m=+0.389410774 container attach 29e635e6fe926d0b481e1122cd4cd7c5e7a45472b11d1e75334cde1ed640df31 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-04-22T15:46:46.372 INFO:journalctl@ceph.osd.3.smithi145.stdout:Apr 22 15:46:46 smithi145 podman[135391]: 2024-04-22 15:46:46.092943977 +0000 UTC m=+0.597363683 container died 29e635e6fe926d0b481e1122cd4cd7c5e7a45472b11d1e75334cde1ed640df31 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-04-22T15:46:46.590 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:46:46 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4[76239]: 2024-04-22T15:46:46.280+0000 7ff5b6a73700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.158272+0000 front 2024-04-22T15:46:18.158290+0000 (oldest deadline 2024-04-22T15:46:42.858188+0000) 2024-04-22T15:46:46.590 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:46:46 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:46:46.398+0000 7fc58549f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:17.214788+0000 front 2024-04-22T15:46:17.214725+0000 (oldest deadline 2024-04-22T15:46:41.914579+0000) 2024-04-22T15:46:46.665 DEBUG:teuthology.orchestra.run.smithi145:> sudo pkill -f 'journalctl -f -n 0 -u ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.3.service' 2024-04-22T15:46:46.702 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T15:46:46.703 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-22T15:46:46.703 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-04-22T15:46:46.703 DEBUG:teuthology.orchestra.run.smithi172:> sudo systemctl stop ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.4 2024-04-22T15:46:46.843 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:46:46 smithi172 systemd[1]: Stopping Ceph osd.4 for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:46:46.844 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:46:46 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:46:46.586+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:19.680121+0000 front 2024-04-22T15:46:19.680173+0000 (oldest deadline 2024-04-22T15:46:44.979938+0000) 2024-04-22T15:46:47.217 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:46:46 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:46:46.840+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:46:47.218 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:46:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4[76239]: 2024-04-22T15:46:47.015+0000 7ff5ba47f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:46:47.218 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:46:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4[76239]: 2024-04-22T15:46:47.015+0000 7ff5ba47f700 -1 osd.4 161 *** Got signal Terminated *** 2024-04-22T15:46:47.218 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:46:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4[76239]: 2024-04-22T15:46:47.015+0000 7ff5ba47f700 -1 osd.4 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T15:46:47.604 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:46:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4[76239]: 2024-04-22T15:46:47.287+0000 7ff5b6a73700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.158272+0000 front 2024-04-22T15:46:18.158290+0000 (oldest deadline 2024-04-22T15:46:42.858188+0000) 2024-04-22T15:46:47.604 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:46:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:46:47.386+0000 7fc58549f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:17.214788+0000 front 2024-04-22T15:46:17.214725+0000 (oldest deadline 2024-04-22T15:46:41.914579+0000) 2024-04-22T15:46:47.869 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:46:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:46:47.601+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:19.680121+0000 front 2024-04-22T15:46:19.680173+0000 (oldest deadline 2024-04-22T15:46:44.979938+0000) 2024-04-22T15:46:48.217 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:46:47 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:46:47.867+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:46:48.624 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:46:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4[76239]: 2024-04-22T15:46:48.304+0000 7ff5b6a73700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.158272+0000 front 2024-04-22T15:46:18.158290+0000 (oldest deadline 2024-04-22T15:46:42.858188+0000) 2024-04-22T15:46:48.625 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:46:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:46:48.425+0000 7fc58549f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:17.214788+0000 front 2024-04-22T15:46:17.214725+0000 (oldest deadline 2024-04-22T15:46:41.914579+0000) 2024-04-22T15:46:48.916 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:46:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:46:48.622+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:19.680121+0000 front 2024-04-22T15:46:19.680173+0000 (oldest deadline 2024-04-22T15:46:44.979938+0000) 2024-04-22T15:46:49.217 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:46:48 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:46:48.913+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:46:49.717 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:46:49 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:46:49.584+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:19.680121+0000 front 2024-04-22T15:46:19.680173+0000 (oldest deadline 2024-04-22T15:46:44.979938+0000) 2024-04-22T15:46:49.718 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:46:49 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4[76239]: 2024-04-22T15:46:49.347+0000 7ff5b6a73700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.158272+0000 front 2024-04-22T15:46:18.158290+0000 (oldest deadline 2024-04-22T15:46:42.858188+0000) 2024-04-22T15:46:49.718 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:46:49 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:46:49.397+0000 7fc58549f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:17.214788+0000 front 2024-04-22T15:46:17.214725+0000 (oldest deadline 2024-04-22T15:46:41.914579+0000) 2024-04-22T15:46:50.217 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:46:49 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:46:49.950+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:46:50.632 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:46:50 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4[76239]: 2024-04-22T15:46:50.299+0000 7ff5b6a73700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.158272+0000 front 2024-04-22T15:46:18.158290+0000 (oldest deadline 2024-04-22T15:46:42.858188+0000) 2024-04-22T15:46:50.633 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:46:50 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:46:50.446+0000 7fc58549f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:17.214788+0000 front 2024-04-22T15:46:17.214725+0000 (oldest deadline 2024-04-22T15:46:41.914579+0000) 2024-04-22T15:46:50.633 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:46:50 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:46:50.446+0000 7fc58549f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.217095+0000 front 2024-04-22T15:46:28.217016+0000 (oldest deadline 2024-04-22T15:46:49.916925+0000) 2024-04-22T15:46:50.966 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:46:50 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:46:50.628+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:19.680121+0000 front 2024-04-22T15:46:19.680173+0000 (oldest deadline 2024-04-22T15:46:44.979938+0000) 2024-04-22T15:46:51.217 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:46:50 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:46:50.962+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:46:51.217 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:46:50 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:46:50.962+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.071016+0000 front 2024-04-22T15:46:28.071060+0000 (oldest deadline 2024-04-22T15:46:50.370830+0000) 2024-04-22T15:46:51.598 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:46:51 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4[76239]: 2024-04-22T15:46:51.272+0000 7ff5b6a73700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.158272+0000 front 2024-04-22T15:46:18.158290+0000 (oldest deadline 2024-04-22T15:46:42.858188+0000) 2024-04-22T15:46:51.598 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:46:51 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4[76239]: 2024-04-22T15:46:51.272+0000 7ff5b6a73700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:26.359305+0000 front 2024-04-22T15:46:26.359320+0000 (oldest deadline 2024-04-22T15:46:50.459087+0000) 2024-04-22T15:46:51.598 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:46:51 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:46:51.408+0000 7fc58549f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:17.214788+0000 front 2024-04-22T15:46:17.214725+0000 (oldest deadline 2024-04-22T15:46:41.914579+0000) 2024-04-22T15:46:51.598 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:46:51 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:46:51.408+0000 7fc58549f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.217095+0000 front 2024-04-22T15:46:28.217016+0000 (oldest deadline 2024-04-22T15:46:49.916925+0000) 2024-04-22T15:46:51.967 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:46:51 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:46:51.595+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:19.680121+0000 front 2024-04-22T15:46:19.680173+0000 (oldest deadline 2024-04-22T15:46:44.979938+0000) 2024-04-22T15:46:52.441 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:46:52 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:46:52.005+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:46:52.441 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:46:52 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:46:52.005+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.071016+0000 front 2024-04-22T15:46:28.071060+0000 (oldest deadline 2024-04-22T15:46:50.370830+0000) 2024-04-22T15:46:52.717 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:46:52 smithi172 podman[100167]: 2024-04-22 15:46:52.457541259 +0000 UTC m=+5.562034137 container died 7ac5776bc07a29bde3f418fa8503a5fbb64dbf4c7c2e9a935e7a1f9889ca5842 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416) 2024-04-22T15:46:52.718 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:46:52 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:46:52.438+0000 7fc58549f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:17.214788+0000 front 2024-04-22T15:46:17.214725+0000 (oldest deadline 2024-04-22T15:46:41.914579+0000) 2024-04-22T15:46:52.718 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:46:52 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:46:52.438+0000 7fc58549f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.217095+0000 front 2024-04-22T15:46:28.217016+0000 (oldest deadline 2024-04-22T15:46:49.916925+0000) 2024-04-22T15:46:52.718 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:46:52 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:46:52.604+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:19.680121+0000 front 2024-04-22T15:46:19.680173+0000 (oldest deadline 2024-04-22T15:46:44.979938+0000) 2024-04-22T15:46:52.718 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:46:52 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:46:52.604+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:27.781520+0000 front 2024-04-22T15:46:27.781404+0000 (oldest deadline 2024-04-22T15:46:51.881253+0000) 2024-04-22T15:46:53.203 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:46:52 smithi172 podman[100167]: 2024-04-22 15:46:52.940823976 +0000 UTC m=+6.045316831 container cleanup 7ac5776bc07a29bde3f418fa8503a5fbb64dbf4c7c2e9a935e7a1f9889ca5842 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-04-22T15:46:53.203 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:46:52 smithi172 bash[100167]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4 2024-04-22T15:46:53.203 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:46:52 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:46:52.981+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:46:53.204 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:46:52 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:46:52.981+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.071016+0000 front 2024-04-22T15:46:28.071060+0000 (oldest deadline 2024-04-22T15:46:50.370830+0000) 2024-04-22T15:46:53.467 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:46:53 smithi172 podman[100184]: 2024-04-22 15:46:53.200873425 +0000 UTC m=+0.741847058 container remove 7ac5776bc07a29bde3f418fa8503a5fbb64dbf4c7c2e9a935e7a1f9889ca5842 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-04-22T15:46:53.468 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:46:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:46:53.415+0000 7fc58549f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:17.214788+0000 front 2024-04-22T15:46:17.214725+0000 (oldest deadline 2024-04-22T15:46:41.914579+0000) 2024-04-22T15:46:53.468 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:46:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:46:53.415+0000 7fc58549f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.217095+0000 front 2024-04-22T15:46:28.217016+0000 (oldest deadline 2024-04-22T15:46:49.916925+0000) 2024-04-22T15:46:53.815 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:46:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:46:53.589+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:19.680121+0000 front 2024-04-22T15:46:19.680173+0000 (oldest deadline 2024-04-22T15:46:44.979938+0000) 2024-04-22T15:46:53.815 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:46:53 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:46:53.589+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:27.781520+0000 front 2024-04-22T15:46:27.781404+0000 (oldest deadline 2024-04-22T15:46:51.881253+0000) 2024-04-22T15:46:53.815 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:46:53 smithi172 podman[100280]: 2024-04-22 15:46:53.56029704 +0000 UTC m=+0.105174377 container create 57f4cb56017870b757306634b9f264b8e8cac5e477e7b179766ef66aa7ed758f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-22T15:46:53.816 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:46:53 smithi172 podman[100280]: 2024-04-22 15:46:53.47296808 +0000 UTC m=+0.017845395 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:46:53.816 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:46:53 smithi172 podman[100280]: 2024-04-22 15:46:53.7378144 +0000 UTC m=+0.282691691 container init 57f4cb56017870b757306634b9f264b8e8cac5e477e7b179766ef66aa7ed758f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-04-22T15:46:53.816 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:46:53 smithi172 podman[100280]: 2024-04-22 15:46:53.741101204 +0000 UTC m=+0.285978502 container start 57f4cb56017870b757306634b9f264b8e8cac5e477e7b179766ef66aa7ed758f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4-deactivate, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_BRANCH=HEAD) 2024-04-22T15:46:53.816 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:46:53 smithi172 podman[100280]: 2024-04-22 15:46:53.812711777 +0000 UTC m=+0.357589085 container attach 57f4cb56017870b757306634b9f264b8e8cac5e477e7b179766ef66aa7ed758f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240416, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-04-22T15:46:53.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:46:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[75967]: ts=2024-04-22T15:46:53.558Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi145:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi172.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.172:8443: connect: connection refused" 2024-04-22T15:46:53.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:46:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[75967]: ts=2024-04-22T15:46:53.559Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi145:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-04-22T15:46:53.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:46:53 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[75967]: ts=2024-04-22T15:46:53.560Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi172.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.172:8443: connect: connection refused" 2024-04-22T15:46:54.218 INFO:journalctl@ceph.osd.4.smithi172.stdout:Apr 22 15:46:54 smithi172 podman[100280]: 2024-04-22 15:46:54.033165147 +0000 UTC m=+0.578042444 container died 57f4cb56017870b757306634b9f264b8e8cac5e477e7b179766ef66aa7ed758f (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-22T15:46:54.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:46:54 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:46:54.007+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:46:54.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:46:54 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:46:54.007+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.071016+0000 front 2024-04-22T15:46:28.071060+0000 (oldest deadline 2024-04-22T15:46:50.370830+0000) 2024-04-22T15:46:54.556 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:46:54 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:46:54.448+0000 7fc58549f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:17.214788+0000 front 2024-04-22T15:46:17.214725+0000 (oldest deadline 2024-04-22T15:46:41.914579+0000) 2024-04-22T15:46:54.556 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:46:54 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:46:54.448+0000 7fc58549f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.217095+0000 front 2024-04-22T15:46:28.217016+0000 (oldest deadline 2024-04-22T15:46:49.916925+0000) 2024-04-22T15:46:54.720 DEBUG:teuthology.orchestra.run.smithi172:> sudo pkill -f 'journalctl -f -n 0 -u ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.4.service' 2024-04-22T15:46:54.758 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T15:46:54.758 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-04-22T15:46:54.758 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-22T15:46:54.758 DEBUG:teuthology.orchestra.run.smithi172:> sudo systemctl stop ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.5 2024-04-22T15:46:54.818 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:46:54 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:46:54.553+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:19.680121+0000 front 2024-04-22T15:46:19.680173+0000 (oldest deadline 2024-04-22T15:46:44.979938+0000) 2024-04-22T15:46:54.818 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:46:54 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:46:54.553+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:27.781520+0000 front 2024-04-22T15:46:27.781404+0000 (oldest deadline 2024-04-22T15:46:51.881253+0000) 2024-04-22T15:46:55.105 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:46:55 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:46:54.998+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:46:55.106 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:46:55 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:46:54.999+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.071016+0000 front 2024-04-22T15:46:28.071060+0000 (oldest deadline 2024-04-22T15:46:50.370830+0000) 2024-04-22T15:46:55.106 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:46:54 smithi172 systemd[1]: Stopping Ceph osd.5 for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:46:55.106 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:46:55 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:46:55.102+0000 7fc58ceb3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:46:55.106 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:46:55 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:46:55.102+0000 7fc58ceb3700 -1 osd.5 161 *** Got signal Terminated *** 2024-04-22T15:46:55.106 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:46:55 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:46:55.102+0000 7fc58ceb3700 -1 osd.5 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T15:46:55.717 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:46:55 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:46:55.458+0000 7fc58549f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:17.214788+0000 front 2024-04-22T15:46:17.214725+0000 (oldest deadline 2024-04-22T15:46:41.914579+0000) 2024-04-22T15:46:55.717 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:46:55 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:46:55.458+0000 7fc58549f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.217095+0000 front 2024-04-22T15:46:28.217016+0000 (oldest deadline 2024-04-22T15:46:49.916925+0000) 2024-04-22T15:46:55.718 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:46:55 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:46:55.585+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:19.680121+0000 front 2024-04-22T15:46:19.680173+0000 (oldest deadline 2024-04-22T15:46:44.979938+0000) 2024-04-22T15:46:55.718 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:46:55 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:46:55.585+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:27.781520+0000 front 2024-04-22T15:46:27.781404+0000 (oldest deadline 2024-04-22T15:46:51.881253+0000) 2024-04-22T15:46:56.467 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:46:56 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:46:56.032+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:46:56.467 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:46:56 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:46:56.032+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.071016+0000 front 2024-04-22T15:46:28.071060+0000 (oldest deadline 2024-04-22T15:46:50.370830+0000) 2024-04-22T15:46:56.967 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:46:56 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:46:56.473+0000 7fc58549f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:17.214788+0000 front 2024-04-22T15:46:17.214725+0000 (oldest deadline 2024-04-22T15:46:41.914579+0000) 2024-04-22T15:46:56.967 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:46:56 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:46:56.473+0000 7fc58549f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.217095+0000 front 2024-04-22T15:46:28.217016+0000 (oldest deadline 2024-04-22T15:46:49.916925+0000) 2024-04-22T15:46:56.968 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:46:56 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:46:56.583+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:19.680121+0000 front 2024-04-22T15:46:19.680173+0000 (oldest deadline 2024-04-22T15:46:44.979938+0000) 2024-04-22T15:46:56.968 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:46:56 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:46:56.583+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:27.781520+0000 front 2024-04-22T15:46:27.781404+0000 (oldest deadline 2024-04-22T15:46:51.881253+0000) 2024-04-22T15:46:57.467 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:46:56 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:46:56.985+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:46:57.468 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:46:56 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:46:56.985+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.071016+0000 front 2024-04-22T15:46:28.071060+0000 (oldest deadline 2024-04-22T15:46:50.370830+0000) 2024-04-22T15:46:57.967 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:46:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:46:57.534+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:19.680121+0000 front 2024-04-22T15:46:19.680173+0000 (oldest deadline 2024-04-22T15:46:44.979938+0000) 2024-04-22T15:46:57.968 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:46:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:46:57.534+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:27.781520+0000 front 2024-04-22T15:46:27.781404+0000 (oldest deadline 2024-04-22T15:46:51.881253+0000) 2024-04-22T15:46:57.968 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:46:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:46:57.534+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-04-22T15:46:34.682977+0000 front 2024-04-22T15:46:34.682911+0000 (oldest deadline 2024-04-22T15:46:56.982828+0000) 2024-04-22T15:46:57.968 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:46:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:46:57.482+0000 7fc58549f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:17.214788+0000 front 2024-04-22T15:46:17.214725+0000 (oldest deadline 2024-04-22T15:46:41.914579+0000) 2024-04-22T15:46:57.968 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:46:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:46:57.482+0000 7fc58549f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.217095+0000 front 2024-04-22T15:46:28.217016+0000 (oldest deadline 2024-04-22T15:46:49.916925+0000) 2024-04-22T15:46:58.466 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:46:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:46:57.970+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:46:58.466 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:46:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:46:57.970+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.071016+0000 front 2024-04-22T15:46:28.071060+0000 (oldest deadline 2024-04-22T15:46:50.370830+0000) 2024-04-22T15:46:58.466 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:46:57 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:46:57.970+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-04-22T15:46:34.972444+0000 front 2024-04-22T15:46:34.972541+0000 (oldest deadline 2024-04-22T15:46:57.272362+0000) 2024-04-22T15:46:58.717 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:46:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:46:58.463+0000 7fc58549f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:17.214788+0000 front 2024-04-22T15:46:17.214725+0000 (oldest deadline 2024-04-22T15:46:41.914579+0000) 2024-04-22T15:46:58.717 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:46:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:46:58.463+0000 7fc58549f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.217095+0000 front 2024-04-22T15:46:28.217016+0000 (oldest deadline 2024-04-22T15:46:49.916925+0000) 2024-04-22T15:46:58.718 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:46:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:46:58.529+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:19.680121+0000 front 2024-04-22T15:46:19.680173+0000 (oldest deadline 2024-04-22T15:46:44.979938+0000) 2024-04-22T15:46:58.718 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:46:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:46:58.529+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:27.781520+0000 front 2024-04-22T15:46:27.781404+0000 (oldest deadline 2024-04-22T15:46:51.881253+0000) 2024-04-22T15:46:58.718 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:46:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:46:58.529+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-04-22T15:46:34.682977+0000 front 2024-04-22T15:46:34.682911+0000 (oldest deadline 2024-04-22T15:46:56.982828+0000) 2024-04-22T15:46:59.217 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:46:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:46:58.953+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:46:59.217 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:46:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:46:58.953+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.071016+0000 front 2024-04-22T15:46:28.071060+0000 (oldest deadline 2024-04-22T15:46:50.370830+0000) 2024-04-22T15:46:59.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:46:58 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:46:58.953+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-04-22T15:46:34.972444+0000 front 2024-04-22T15:46:34.972541+0000 (oldest deadline 2024-04-22T15:46:57.272362+0000) 2024-04-22T15:46:59.717 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:46:59 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:46:59.434+0000 7fc58549f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:17.214788+0000 front 2024-04-22T15:46:17.214725+0000 (oldest deadline 2024-04-22T15:46:41.914579+0000) 2024-04-22T15:46:59.718 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:46:59 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5[80940]: 2024-04-22T15:46:59.434+0000 7fc58549f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.217095+0000 front 2024-04-22T15:46:28.217016+0000 (oldest deadline 2024-04-22T15:46:49.916925+0000) 2024-04-22T15:46:59.718 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:46:59 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:46:59.507+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:19.680121+0000 front 2024-04-22T15:46:19.680173+0000 (oldest deadline 2024-04-22T15:46:44.979938+0000) 2024-04-22T15:46:59.718 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:46:59 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:46:59.508+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:27.781520+0000 front 2024-04-22T15:46:27.781404+0000 (oldest deadline 2024-04-22T15:46:51.881253+0000) 2024-04-22T15:46:59.718 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:46:59 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:46:59.508+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-04-22T15:46:34.682977+0000 front 2024-04-22T15:46:34.682911+0000 (oldest deadline 2024-04-22T15:46:56.982828+0000) 2024-04-22T15:47:00.217 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:46:59 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:46:59.927+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:47:00.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:46:59 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:46:59.927+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.071016+0000 front 2024-04-22T15:46:28.071060+0000 (oldest deadline 2024-04-22T15:46:50.370830+0000) 2024-04-22T15:47:00.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:46:59 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:46:59.927+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-04-22T15:46:34.972444+0000 front 2024-04-22T15:46:34.972541+0000 (oldest deadline 2024-04-22T15:46:57.272362+0000) 2024-04-22T15:47:00.892 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:47:00 smithi172 podman[100397]: 2024-04-22 15:47:00.627576818 +0000 UTC m=+5.634896752 container died 5f803668a17b6fdff528ee806b9e6ff89fbb65b4877b300b5c329905f57179b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2) 2024-04-22T15:47:00.892 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:00 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:47:00.541+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:19.680121+0000 front 2024-04-22T15:46:19.680173+0000 (oldest deadline 2024-04-22T15:46:44.979938+0000) 2024-04-22T15:47:00.892 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:00 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:47:00.542+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:27.781520+0000 front 2024-04-22T15:46:27.781404+0000 (oldest deadline 2024-04-22T15:46:51.881253+0000) 2024-04-22T15:47:00.892 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:00 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:47:00.542+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-04-22T15:46:34.682977+0000 front 2024-04-22T15:46:34.682911+0000 (oldest deadline 2024-04-22T15:46:56.982828+0000) 2024-04-22T15:47:01.217 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:47:00 smithi172 podman[100397]: 2024-04-22 15:47:00.980398674 +0000 UTC m=+5.987718607 container cleanup 5f803668a17b6fdff528ee806b9e6ff89fbb65b4877b300b5c329905f57179b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5, org.label-schema.build-date=20240416, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-04-22T15:47:01.217 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:47:00 smithi172 bash[100397]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5 2024-04-22T15:47:01.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:00 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:00.945+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:47:01.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:00 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:00.945+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.071016+0000 front 2024-04-22T15:46:28.071060+0000 (oldest deadline 2024-04-22T15:46:50.370830+0000) 2024-04-22T15:47:01.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:00 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:00.945+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-04-22T15:46:34.972444+0000 front 2024-04-22T15:46:34.972541+0000 (oldest deadline 2024-04-22T15:46:57.272362+0000) 2024-04-22T15:47:01.518 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:47:01 smithi172 podman[100415]: 2024-04-22 15:47:01.24800992 +0000 UTC m=+0.619105176 container remove 5f803668a17b6fdff528ee806b9e6ff89fbb65b4877b300b5c329905f57179b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-04-22T15:47:01.836 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:47:01.514+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:19.680121+0000 front 2024-04-22T15:46:19.680173+0000 (oldest deadline 2024-04-22T15:46:44.979938+0000) 2024-04-22T15:47:01.837 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:47:01.514+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:27.781520+0000 front 2024-04-22T15:46:27.781404+0000 (oldest deadline 2024-04-22T15:46:51.881253+0000) 2024-04-22T15:47:01.837 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:47:01.514+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-04-22T15:46:34.682977+0000 front 2024-04-22T15:46:34.682911+0000 (oldest deadline 2024-04-22T15:46:56.982828+0000) 2024-04-22T15:47:01.837 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:47:01 smithi172 podman[100505]: 2024-04-22 15:47:01.455386458 +0000 UTC m=+0.018702728 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:47:01.837 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:47:01 smithi172 podman[100505]: 2024-04-22 15:47:01.557315987 +0000 UTC m=+0.120632225 container create e6b0c9d5d74fc971055dce2aa5f1602847e82e02344beb6f47c44943bb54708e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.build-date=20240416, ceph=True, org.label-schema.vendor=CentOS) 2024-04-22T15:47:01.837 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:47:01 smithi172 podman[100505]: 2024-04-22 15:47:01.834682377 +0000 UTC m=+0.397998621 container init e6b0c9d5d74fc971055dce2aa5f1602847e82e02344beb6f47c44943bb54708e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.build-date=20240416, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True) 2024-04-22T15:47:02.129 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:47:01 smithi172 podman[100505]: 2024-04-22 15:47:01.83777893 +0000 UTC m=+0.401095174 container start e6b0c9d5d74fc971055dce2aa5f1602847e82e02344beb6f47c44943bb54708e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-04-22T15:47:02.129 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:47:01 smithi172 podman[100505]: 2024-04-22 15:47:01.909654538 +0000 UTC m=+0.472970784 container attach e6b0c9d5d74fc971055dce2aa5f1602847e82e02344beb6f47c44943bb54708e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240416, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-04-22T15:47:02.130 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:01.915+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:47:02.130 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:01.915+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.071016+0000 front 2024-04-22T15:46:28.071060+0000 (oldest deadline 2024-04-22T15:46:50.370830+0000) 2024-04-22T15:47:02.130 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:01 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:01.915+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-04-22T15:46:34.972444+0000 front 2024-04-22T15:46:34.972541+0000 (oldest deadline 2024-04-22T15:46:57.272362+0000) 2024-04-22T15:47:02.414 INFO:journalctl@ceph.osd.5.smithi172.stdout:Apr 22 15:47:02 smithi172 podman[100505]: 2024-04-22 15:47:02.12767072 +0000 UTC m=+0.690986973 container died e6b0c9d5d74fc971055dce2aa5f1602847e82e02344beb6f47c44943bb54708e (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True) 2024-04-22T15:47:02.665 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:02 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:47:02.560+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:19.680121+0000 front 2024-04-22T15:46:19.680173+0000 (oldest deadline 2024-04-22T15:46:44.979938+0000) 2024-04-22T15:47:02.665 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:02 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:47:02.560+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:27.781520+0000 front 2024-04-22T15:46:27.781404+0000 (oldest deadline 2024-04-22T15:46:51.881253+0000) 2024-04-22T15:47:02.665 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:02 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:47:02.560+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-04-22T15:46:34.682977+0000 front 2024-04-22T15:46:34.682911+0000 (oldest deadline 2024-04-22T15:46:56.982828+0000) 2024-04-22T15:47:02.681 DEBUG:teuthology.orchestra.run.smithi172:> sudo pkill -f 'journalctl -f -n 0 -u ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.5.service' 2024-04-22T15:47:02.758 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T15:47:02.759 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-22T15:47:02.759 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-04-22T15:47:02.759 DEBUG:teuthology.orchestra.run.smithi172:> sudo systemctl stop ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.6 2024-04-22T15:47:02.915 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:02 smithi172 systemd[1]: Stopping Ceph osd.6 for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:47:02.916 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:02 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:02.912+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:47:02.916 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:02 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:02.912+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.071016+0000 front 2024-04-22T15:46:28.071060+0000 (oldest deadline 2024-04-22T15:46:50.370830+0000) 2024-04-22T15:47:02.916 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:02 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:02.912+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-04-22T15:46:34.972444+0000 front 2024-04-22T15:46:34.972541+0000 (oldest deadline 2024-04-22T15:46:57.272362+0000) 2024-04-22T15:47:03.217 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:03 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:47:03.082+0000 7f9459ed8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:47:03.217 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:03 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:47:03.082+0000 7f9459ed8700 -1 osd.6 161 *** Got signal Terminated *** 2024-04-22T15:47:03.217 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:03 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:47:03.082+0000 7f9459ed8700 -1 osd.6 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T15:47:03.873 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:47:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[75967]: ts=2024-04-22T15:47:03.560Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi172.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.172:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi145:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-04-22T15:47:03.873 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:47:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[75967]: ts=2024-04-22T15:47:03.560Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi145:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-04-22T15:47:03.873 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:47:03 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[75967]: ts=2024-04-22T15:47:03.561Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi172.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.172:8443: connect: connection refused" 2024-04-22T15:47:03.911 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:03 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:47:03.608+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:19.680121+0000 front 2024-04-22T15:46:19.680173+0000 (oldest deadline 2024-04-22T15:46:44.979938+0000) 2024-04-22T15:47:03.912 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:03 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:47:03.608+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:27.781520+0000 front 2024-04-22T15:46:27.781404+0000 (oldest deadline 2024-04-22T15:46:51.881253+0000) 2024-04-22T15:47:03.912 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:03 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:47:03.608+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-04-22T15:46:34.682977+0000 front 2024-04-22T15:46:34.682911+0000 (oldest deadline 2024-04-22T15:46:56.982828+0000) 2024-04-22T15:47:04.217 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:03 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:03.908+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:47:04.217 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:03 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:03.908+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.071016+0000 front 2024-04-22T15:46:28.071060+0000 (oldest deadline 2024-04-22T15:46:50.370830+0000) 2024-04-22T15:47:04.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:03 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:03.908+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-04-22T15:46:34.972444+0000 front 2024-04-22T15:46:34.972541+0000 (oldest deadline 2024-04-22T15:46:57.272362+0000) 2024-04-22T15:47:04.917 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:04 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:47:04.564+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:19.680121+0000 front 2024-04-22T15:46:19.680173+0000 (oldest deadline 2024-04-22T15:46:44.979938+0000) 2024-04-22T15:47:04.917 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:04 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:47:04.564+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:27.781520+0000 front 2024-04-22T15:46:27.781404+0000 (oldest deadline 2024-04-22T15:46:51.881253+0000) 2024-04-22T15:47:04.917 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:04 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:47:04.564+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-04-22T15:46:34.682977+0000 front 2024-04-22T15:46:34.682911+0000 (oldest deadline 2024-04-22T15:46:56.982828+0000) 2024-04-22T15:47:05.217 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:04 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:04.914+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:47:05.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:04 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:04.914+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.071016+0000 front 2024-04-22T15:46:28.071060+0000 (oldest deadline 2024-04-22T15:46:50.370830+0000) 2024-04-22T15:47:05.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:04 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:04.914+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-04-22T15:46:34.972444+0000 front 2024-04-22T15:46:34.972541+0000 (oldest deadline 2024-04-22T15:46:57.272362+0000) 2024-04-22T15:47:05.870 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:47:05.576+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:19.680121+0000 front 2024-04-22T15:46:19.680173+0000 (oldest deadline 2024-04-22T15:46:44.979938+0000) 2024-04-22T15:47:05.871 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:47:05.576+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:27.781520+0000 front 2024-04-22T15:46:27.781404+0000 (oldest deadline 2024-04-22T15:46:51.881253+0000) 2024-04-22T15:47:05.871 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:47:05.576+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-04-22T15:46:34.682977+0000 front 2024-04-22T15:46:34.682911+0000 (oldest deadline 2024-04-22T15:46:56.982828+0000) 2024-04-22T15:47:06.217 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:05.867+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:47:06.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:05.867+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.071016+0000 front 2024-04-22T15:46:28.071060+0000 (oldest deadline 2024-04-22T15:46:50.370830+0000) 2024-04-22T15:47:06.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:05 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:05.867+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-04-22T15:46:34.972444+0000 front 2024-04-22T15:46:34.972541+0000 (oldest deadline 2024-04-22T15:46:57.272362+0000) 2024-04-22T15:47:06.903 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:06 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:47:06.612+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:19.680121+0000 front 2024-04-22T15:46:19.680173+0000 (oldest deadline 2024-04-22T15:46:44.979938+0000) 2024-04-22T15:47:06.904 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:06 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:47:06.612+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:27.781520+0000 front 2024-04-22T15:46:27.781404+0000 (oldest deadline 2024-04-22T15:46:51.881253+0000) 2024-04-22T15:47:06.904 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:06 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:47:06.612+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-04-22T15:46:34.682977+0000 front 2024-04-22T15:46:34.682911+0000 (oldest deadline 2024-04-22T15:46:56.982828+0000) 2024-04-22T15:47:06.904 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:06 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:47:06.612+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6830 osd.3 since back 2024-04-22T15:46:43.884412+0000 front 2024-04-22T15:46:43.884512+0000 (oldest deadline 2024-04-22T15:47:06.084659+0000) 2024-04-22T15:47:07.217 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:06 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:06.900+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:47:07.217 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:06 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:06.900+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.071016+0000 front 2024-04-22T15:46:28.071060+0000 (oldest deadline 2024-04-22T15:46:50.370830+0000) 2024-04-22T15:47:07.217 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:06 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:06.900+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-04-22T15:46:34.972444+0000 front 2024-04-22T15:46:34.972541+0000 (oldest deadline 2024-04-22T15:46:57.272362+0000) 2024-04-22T15:47:07.217 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:06 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:06.900+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6830 osd.3 since back 2024-04-22T15:46:38.973237+0000 front 2024-04-22T15:46:38.973222+0000 (oldest deadline 2024-04-22T15:47:05.973521+0000) 2024-04-22T15:47:07.911 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:07 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:47:07.583+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:19.680121+0000 front 2024-04-22T15:46:19.680173+0000 (oldest deadline 2024-04-22T15:46:44.979938+0000) 2024-04-22T15:47:07.911 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:07 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:47:07.583+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:27.781520+0000 front 2024-04-22T15:46:27.781404+0000 (oldest deadline 2024-04-22T15:46:51.881253+0000) 2024-04-22T15:47:07.911 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:07 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:47:07.583+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-04-22T15:46:34.682977+0000 front 2024-04-22T15:46:34.682911+0000 (oldest deadline 2024-04-22T15:46:56.982828+0000) 2024-04-22T15:47:07.911 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:07 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6[85957]: 2024-04-22T15:47:07.583+0000 7f94564cc700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.145:6830 osd.3 since back 2024-04-22T15:46:43.884412+0000 front 2024-04-22T15:46:43.884512+0000 (oldest deadline 2024-04-22T15:47:06.084659+0000) 2024-04-22T15:47:08.217 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:07 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:07.908+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:47:08.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:07 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:07.908+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.071016+0000 front 2024-04-22T15:46:28.071060+0000 (oldest deadline 2024-04-22T15:46:50.370830+0000) 2024-04-22T15:47:08.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:07 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:07.908+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-04-22T15:46:34.972444+0000 front 2024-04-22T15:46:34.972541+0000 (oldest deadline 2024-04-22T15:46:57.272362+0000) 2024-04-22T15:47:08.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:07 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:07.908+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6830 osd.3 since back 2024-04-22T15:46:38.973237+0000 front 2024-04-22T15:46:38.973222+0000 (oldest deadline 2024-04-22T15:47:05.973521+0000) 2024-04-22T15:47:08.810 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:08 smithi172 podman[100610]: 2024-04-22 15:47:08.534289676 +0000 UTC m=+5.556323274 container died 9e049dd4678d36462e6608f5514e745938cf46a353247b51b627458aeb62b076 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-04-22T15:47:09.127 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:08 smithi172 podman[100610]: 2024-04-22 15:47:08.898928781 +0000 UTC m=+5.920962375 container cleanup 9e049dd4678d36462e6608f5514e745938cf46a353247b51b627458aeb62b076 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, maintainer=Guillaume Abrioux ) 2024-04-22T15:47:09.127 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:08 smithi172 bash[100610]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6 2024-04-22T15:47:09.127 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:08 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:08.890+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:47:09.127 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:08 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:08.890+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.071016+0000 front 2024-04-22T15:46:28.071060+0000 (oldest deadline 2024-04-22T15:46:50.370830+0000) 2024-04-22T15:47:09.127 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:08 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:08.890+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-04-22T15:46:34.972444+0000 front 2024-04-22T15:46:34.972541+0000 (oldest deadline 2024-04-22T15:46:57.272362+0000) 2024-04-22T15:47:09.127 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:08 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:08.890+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6830 osd.3 since back 2024-04-22T15:46:38.973237+0000 front 2024-04-22T15:46:38.973222+0000 (oldest deadline 2024-04-22T15:47:05.973521+0000) 2024-04-22T15:47:09.466 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:09 smithi172 podman[100628]: 2024-04-22 15:47:09.12496212 +0000 UTC m=+0.586022415 container remove 9e049dd4678d36462e6608f5514e745938cf46a353247b51b627458aeb62b076 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240416, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T15:47:09.717 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:09 smithi172 podman[100722]: 2024-04-22 15:47:09.367400048 +0000 UTC m=+0.018287419 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:47:09.718 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:09 smithi172 podman[100722]: 2024-04-22 15:47:09.467439166 +0000 UTC m=+0.118326562 container create cc61596214b32cdba049598156447d62ba1b1e64cfdc6834fedf53be04e924b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-22T15:47:09.718 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:09 smithi172 podman[100722]: 2024-04-22 15:47:09.661690154 +0000 UTC m=+0.312577502 container init cc61596214b32cdba049598156447d62ba1b1e64cfdc6834fedf53be04e924b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6-deactivate, ceph=True, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_CLEAN=True) 2024-04-22T15:47:09.718 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:09 smithi172 podman[100722]: 2024-04-22 15:47:09.664826644 +0000 UTC m=+0.315713997 container start cc61596214b32cdba049598156447d62ba1b1e64cfdc6834fedf53be04e924b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-22T15:47:10.217 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:09 smithi172 podman[100722]: 2024-04-22 15:47:09.744853223 +0000 UTC m=+0.395740578 container attach cc61596214b32cdba049598156447d62ba1b1e64cfdc6834fedf53be04e924b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-22T15:47:10.218 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:09 smithi172 podman[100722]: 2024-04-22 15:47:09.970897235 +0000 UTC m=+0.621784591 container died cc61596214b32cdba049598156447d62ba1b1e64cfdc6834fedf53be04e924b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-04-22T15:47:10.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:09 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:09.910+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:47:10.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:09 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:09.910+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.071016+0000 front 2024-04-22T15:46:28.071060+0000 (oldest deadline 2024-04-22T15:46:50.370830+0000) 2024-04-22T15:47:10.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:09 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:09.910+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-04-22T15:46:34.972444+0000 front 2024-04-22T15:46:34.972541+0000 (oldest deadline 2024-04-22T15:46:57.272362+0000) 2024-04-22T15:47:10.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:09 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:09.910+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6830 osd.3 since back 2024-04-22T15:46:38.973237+0000 front 2024-04-22T15:46:38.973222+0000 (oldest deadline 2024-04-22T15:47:05.973521+0000) 2024-04-22T15:47:10.486 DEBUG:teuthology.orchestra.run.smithi172:> sudo pkill -f 'journalctl -f -n 0 -u ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.6.service' 2024-04-22T15:47:10.507 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:10 smithi172 podman[100748]: 2024-04-22 15:47:10.464812787 +0000 UTC m=+0.485751429 container remove cc61596214b32cdba049598156447d62ba1b1e64cfdc6834fedf53be04e924b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-6-deactivate, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-04-22T15:47:10.507 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:10 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.6.service: Deactivated successfully. 2024-04-22T15:47:10.507 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:10 smithi172 systemd[1]: Stopped Ceph osd.6 for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:47:10.507 INFO:journalctl@ceph.osd.6.smithi172.stdout:Apr 22 15:47:10 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.6.service: Consumed 4.073s CPU time. 2024-04-22T15:47:10.522 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T15:47:10.522 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-04-22T15:47:10.522 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-04-22T15:47:10.522 DEBUG:teuthology.orchestra.run.smithi172:> sudo systemctl stop ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.7 2024-04-22T15:47:10.876 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:10 smithi172 systemd[1]: Stopping Ceph osd.7 for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:47:10.877 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:10 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:10.873+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:47:11.217 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:10 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:10.873+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.071016+0000 front 2024-04-22T15:46:28.071060+0000 (oldest deadline 2024-04-22T15:46:50.370830+0000) 2024-04-22T15:47:11.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:10 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:10.873+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-04-22T15:46:34.972444+0000 front 2024-04-22T15:46:34.972541+0000 (oldest deadline 2024-04-22T15:46:57.272362+0000) 2024-04-22T15:47:11.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:10 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:10.873+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6830 osd.3 since back 2024-04-22T15:46:38.973237+0000 front 2024-04-22T15:46:38.973222+0000 (oldest deadline 2024-04-22T15:47:05.973521+0000) 2024-04-22T15:47:11.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:10 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:10.900+0000 7f5e259e3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T15:47:11.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:10 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:10.900+0000 7f5e259e3700 -1 osd.7 161 *** Got signal Terminated *** 2024-04-22T15:47:11.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:10 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:10.900+0000 7f5e259e3700 -1 osd.7 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T15:47:12.119 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:11 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:11.854+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:47:12.119 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:11 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:11.854+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.071016+0000 front 2024-04-22T15:46:28.071060+0000 (oldest deadline 2024-04-22T15:46:50.370830+0000) 2024-04-22T15:47:12.119 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:11 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:11.854+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-04-22T15:46:34.972444+0000 front 2024-04-22T15:46:34.972541+0000 (oldest deadline 2024-04-22T15:46:57.272362+0000) 2024-04-22T15:47:12.119 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:11 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:11.854+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6830 osd.3 since back 2024-04-22T15:46:38.973237+0000 front 2024-04-22T15:46:38.973222+0000 (oldest deadline 2024-04-22T15:47:05.973521+0000) 2024-04-22T15:47:13.217 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:12.834+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:47:13.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:12.834+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.071016+0000 front 2024-04-22T15:46:28.071060+0000 (oldest deadline 2024-04-22T15:46:50.370830+0000) 2024-04-22T15:47:13.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:12.834+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-04-22T15:46:34.972444+0000 front 2024-04-22T15:46:34.972541+0000 (oldest deadline 2024-04-22T15:46:57.272362+0000) 2024-04-22T15:47:13.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:12 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:12.834+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6830 osd.3 since back 2024-04-22T15:46:38.973237+0000 front 2024-04-22T15:46:38.973222+0000 (oldest deadline 2024-04-22T15:47:05.973521+0000) 2024-04-22T15:47:13.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:47:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[75967]: ts=2024-04-22T15:47:13.561Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi172.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.172:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi145:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-04-22T15:47:13.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:47:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[75967]: ts=2024-04-22T15:47:13.561Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi145:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-04-22T15:47:13.872 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:47:13 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[75967]: ts=2024-04-22T15:47:13.562Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi172.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.172:8443: connect: connection refused" 2024-04-22T15:47:14.217 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:13.816+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:47:14.217 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:13.816+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.071016+0000 front 2024-04-22T15:46:28.071060+0000 (oldest deadline 2024-04-22T15:46:50.370830+0000) 2024-04-22T15:47:14.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:13.816+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-04-22T15:46:34.972444+0000 front 2024-04-22T15:46:34.972541+0000 (oldest deadline 2024-04-22T15:46:57.272362+0000) 2024-04-22T15:47:14.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:13 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:13.816+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6830 osd.3 since back 2024-04-22T15:46:38.973237+0000 front 2024-04-22T15:46:38.973222+0000 (oldest deadline 2024-04-22T15:47:05.973521+0000) 2024-04-22T15:47:15.217 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:14 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:14.850+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:47:15.217 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:14 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:14.850+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.071016+0000 front 2024-04-22T15:46:28.071060+0000 (oldest deadline 2024-04-22T15:46:50.370830+0000) 2024-04-22T15:47:15.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:14 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:14.850+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-04-22T15:46:34.972444+0000 front 2024-04-22T15:46:34.972541+0000 (oldest deadline 2024-04-22T15:46:57.272362+0000) 2024-04-22T15:47:15.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:14 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:14.850+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6830 osd.3 since back 2024-04-22T15:46:38.973237+0000 front 2024-04-22T15:46:38.973222+0000 (oldest deadline 2024-04-22T15:47:05.973521+0000) 2024-04-22T15:47:15.218 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:14 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:14.850+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.172:6804 osd.4 since back 2024-04-22T15:46:51.874444+0000 front 2024-04-22T15:46:51.874418+0000 (oldest deadline 2024-04-22T15:47:14.774382+0000) 2024-04-22T15:47:16.212 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:15 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:15.873+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-04-22T15:46:18.968333+0000 front 2024-04-22T15:46:18.968434+0000 (oldest deadline 2024-04-22T15:46:41.268189+0000) 2024-04-22T15:47:16.212 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:15 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:15.873+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-04-22T15:46:28.071016+0000 front 2024-04-22T15:46:28.071060+0000 (oldest deadline 2024-04-22T15:46:50.370830+0000) 2024-04-22T15:47:16.212 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:15 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:15.873+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-04-22T15:46:34.972444+0000 front 2024-04-22T15:46:34.972541+0000 (oldest deadline 2024-04-22T15:46:57.272362+0000) 2024-04-22T15:47:16.213 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:15 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:15.873+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.145:6830 osd.3 since back 2024-04-22T15:46:38.973237+0000 front 2024-04-22T15:46:38.973222+0000 (oldest deadline 2024-04-22T15:47:05.973521+0000) 2024-04-22T15:47:16.213 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:15 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7[90519]: 2024-04-22T15:47:15.873+0000 7f5e21fd7700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.172:6804 osd.4 since back 2024-04-22T15:46:51.874444+0000 front 2024-04-22T15:46:51.874418+0000 (oldest deadline 2024-04-22T15:47:14.774382+0000) 2024-04-22T15:47:16.467 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:16 smithi172 podman[100828]: 2024-04-22 15:47:16.209723415 +0000 UTC m=+5.470589307 container died 951a8da56333a1527aed1fd5682435f1b4c4856ab74ba527d37a9c99c2f373cf (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240416, org.label-schema.license=GPLv2) 2024-04-22T15:47:16.845 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:16 smithi172 podman[100828]: 2024-04-22 15:47:16.625701804 +0000 UTC m=+5.886567695 container cleanup 951a8da56333a1527aed1fd5682435f1b4c4856ab74ba527d37a9c99c2f373cf (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.build-date=20240416) 2024-04-22T15:47:16.846 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:16 smithi172 bash[100828]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7 2024-04-22T15:47:17.146 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:16 smithi172 podman[100856]: 2024-04-22 15:47:16.843880968 +0000 UTC m=+0.625322417 container remove 951a8da56333a1527aed1fd5682435f1b4c4856ab74ba527d37a9c99c2f373cf (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-04-22T15:47:17.416 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:17 smithi172 podman[100949]: 2024-04-22 15:47:17.144630125 +0000 UTC m=+0.089101497 container create d00ba678d21ecec264f334726fad0d87b81466035439250963e25b5d40c92d96 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240416, ceph=True, GIT_BRANCH=HEAD) 2024-04-22T15:47:17.416 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:17 smithi172 podman[100949]: 2024-04-22 15:47:17.073542307 +0000 UTC m=+0.018013690 image pull 3d983c26e278e01e9b639456b0249f83c9cc4e6f892b34f6e771c11bdf167655 quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb 2024-04-22T15:47:17.416 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:17 smithi172 podman[100949]: 2024-04-22 15:47:17.347675838 +0000 UTC m=+0.292147221 container init d00ba678d21ecec264f334726fad0d87b81466035439250963e25b5d40c92d96 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240416, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49) 2024-04-22T15:47:17.416 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:17 smithi172 podman[100949]: 2024-04-22 15:47:17.351231576 +0000 UTC m=+0.295702955 container start d00ba678d21ecec264f334726fad0d87b81466035439250963e25b5d40c92d96 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7-deactivate, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, ceph=True, org.label-schema.build-date=20240416, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-04-22T15:47:17.416 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:17 smithi172 podman[100949]: 2024-04-22 15:47:17.413592883 +0000 UTC m=+0.358064267 container attach d00ba678d21ecec264f334726fad0d87b81466035439250963e25b5d40c92d96 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240416, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-04-22T15:47:17.717 INFO:journalctl@ceph.osd.7.smithi172.stdout:Apr 22 15:47:17 smithi172 podman[100949]: 2024-04-22 15:47:17.643968947 +0000 UTC m=+0.588440321 container died d00ba678d21ecec264f334726fad0d87b81466035439250963e25b5d40c92d96 (image=quay.ceph.io/ceph-ci/ceph@sha256:723695d45457befe8773a74f1dc5b88a518c622ccec5f4879018f9c82e2c2abb, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-osd-7-deactivate, RELEASE=wip-adk-testing-2024-04-22-0809-5883d49, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240416, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-22T15:47:18.185 DEBUG:teuthology.orchestra.run.smithi172:> sudo pkill -f 'journalctl -f -n 0 -u ceph-869afcfa-00b9-11ef-bc93-c7b262605968@osd.7.service' 2024-04-22T15:47:18.224 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T15:47:18.224 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-04-22T15:47:18.224 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-04-22T15:47:18.224 DEBUG:teuthology.orchestra.run.smithi172:> sudo systemctl stop ceph-869afcfa-00b9-11ef-bc93-c7b262605968@prometheus.a 2024-04-22T15:47:18.467 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:47:18 smithi172 systemd[1]: Stopping Ceph prometheus.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:47:18.823 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:47:18 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:47:18.570Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-22T15:47:18.823 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:47:18 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:47:18.570Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-22T15:47:18.824 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:47:18 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:47:18.570Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-22T15:47:18.824 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:47:18 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:47:18.570Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-22T15:47:18.824 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:47:18 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:47:18.570Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-22T15:47:18.824 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:47:18 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:47:18.570Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-22T15:47:18.824 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:47:18 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:47:18.570Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-22T15:47:18.824 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:47:18 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:47:18.570Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-22T15:47:18.824 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:47:18 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:47:18.570Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-22T15:47:18.824 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:47:18 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:47:18.571Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-22T15:47:18.825 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:47:18 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:47:18.571Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-22T15:47:18.825 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:47:18 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a[61346]: ts=2024-04-22T15:47:18.571Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-22T15:47:18.825 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:47:18 smithi172 podman[101054]: 2024-04-22 15:47:18.580237008 +0000 UTC m=+0.142325245 container died 51cf4df6e8d45c7decc16f595f43cf242f20c56a1634d623e04631b85216f6e8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:47:19.098 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:47:18 smithi172 podman[101054]: 2024-04-22 15:47:18.911693306 +0000 UTC m=+0.473781541 container cleanup 51cf4df6e8d45c7decc16f595f43cf242f20c56a1634d623e04631b85216f6e8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:47:19.098 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:47:18 smithi172 bash[101054]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a 2024-04-22T15:47:19.329 DEBUG:teuthology.orchestra.run.smithi172:> sudo pkill -f 'journalctl -f -n 0 -u ceph-869afcfa-00b9-11ef-bc93-c7b262605968@prometheus.a.service' 2024-04-22T15:47:19.350 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:47:19 smithi172 podman[101068]: 2024-04-22 15:47:19.095803602 +0000 UTC m=+0.517286071 container remove 51cf4df6e8d45c7decc16f595f43cf242f20c56a1634d623e04631b85216f6e8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T15:47:19.350 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:47:19 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-22T15:47:19.350 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:47:19 smithi172 systemd[1]: Stopped Ceph prometheus.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:47:19.350 INFO:journalctl@ceph.prometheus.a.smithi172.stdout:Apr 22 15:47:19 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 4.381s CPU time. 2024-04-22T15:47:19.363 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T15:47:19.363 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-04-22T15:47:19.363 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-22T15:47:21.122 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:47:20 smithi145 systemd[1]: Stopping Ceph node-exporter.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:47:21.531 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:47:21 smithi145 podman[135782]: 2024-04-22 15:47:21.123379555 +0000 UTC m=+0.150743594 container died 8c75c26fec05f244883c540f215d8ad9667cf3608ba7133e8c33937e7a061839 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T15:47:21.531 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:47:21 smithi145 podman[135782]: 2024-04-22 15:47:21.529275148 +0000 UTC m=+0.556639187 container cleanup 8c75c26fec05f244883c540f215d8ad9667cf3608ba7133e8c33937e7a061839 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T15:47:21.872 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:47:21 smithi145 bash[135782]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a 2024-04-22T15:47:21.872 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:47:21 smithi145 podman[135796]: 2024-04-22 15:47:21.730547031 +0000 UTC m=+0.609169968 container remove 8c75c26fec05f244883c540f215d8ad9667cf3608ba7133e8c33937e7a061839 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T15:47:21.872 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:47:21 smithi145 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-22T15:47:22.372 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:47:22 smithi145 systemd[1]: Stopping Ceph alertmanager.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:47:22.373 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:47:21 smithi145 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-22T15:47:22.373 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:47:21 smithi145 systemd[1]: Stopped Ceph node-exporter.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:47:22.373 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Apr 22 15:47:21 smithi145 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@node-exporter.a.service: Consumed 3.395s CPU time. 2024-04-22T15:47:22.729 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:47:22 smithi145 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a[75967]: ts=2024-04-22T15:47:22.383Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-22T15:47:22.729 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:47:22 smithi145 podman[135901]: 2024-04-22 15:47:22.424155748 +0000 UTC m=+0.148809008 container died 1c58ccf37ffd79a19f1e1f23b6570349cfd57ec0e4f0e289d8a172d47e89c0b1 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T15:47:23.029 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:47:22 smithi145 podman[135901]: 2024-04-22 15:47:22.833946252 +0000 UTC m=+0.558599508 container cleanup 1c58ccf37ffd79a19f1e1f23b6570349cfd57ec0e4f0e289d8a172d47e89c0b1 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T15:47:23.029 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:47:22 smithi145 bash[135901]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a 2024-04-22T15:47:23.288 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:47:23 smithi145 podman[135915]: 2024-04-22 15:47:23.026656319 +0000 UTC m=+0.593387373 container remove 1c58ccf37ffd79a19f1e1f23b6570349cfd57ec0e4f0e289d8a172d47e89c0b1 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T15:47:23.288 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:47:23 smithi145 podman[135915]: 2024-04-22 15:47:23.102351814 +0000 UTC m=+0.669082868 volume remove 97b5e88a6b2a7d0eb86406430ccb6b8aeb758f9b485daa2efc8978c209097f67 2024-04-22T15:47:23.622 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:47:23 smithi145 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-22T15:47:23.622 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:47:23 smithi145 systemd[1]: Stopped Ceph alertmanager.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:47:23.622 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Apr 22 15:47:23 smithi145 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@alertmanager.a.service: Consumed 1.641s CPU time. 2024-04-22T15:47:46.419 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-22T15:47:48.137 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:47:47 smithi172 systemd[1]: Stopping Ceph node-exporter.b for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:47:48.467 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:47:48 smithi172 podman[101462]: 2024-04-22 15:47:48.13934216 +0000 UTC m=+0.103433747 container died 8fd2949e2eaf0973ec1c44cca152ec6c96f8b6e9353482fab051598d8411a690 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T15:47:48.962 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:47:48 smithi172 podman[101462]: 2024-04-22 15:47:48.55418278 +0000 UTC m=+0.518274367 container cleanup 8fd2949e2eaf0973ec1c44cca152ec6c96f8b6e9353482fab051598d8411a690 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T15:47:48.962 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:47:48 smithi172 bash[101462]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b 2024-04-22T15:47:48.962 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:47:48 smithi172 podman[101477]: 2024-04-22 15:47:48.738961452 +0000 UTC m=+0.601843392 container remove 8fd2949e2eaf0973ec1c44cca152ec6c96f8b6e9353482fab051598d8411a690 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T15:47:48.962 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:47:48 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-22T15:47:49.385 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:47:49 smithi172 systemd[1]: Stopping Ceph grafana.a for 869afcfa-00b9-11ef-bc93-c7b262605968... 2024-04-22T15:47:49.386 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:47:48 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-22T15:47:49.386 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:47:48 smithi172 systemd[1]: Stopped Ceph node-exporter.b for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:47:49.386 INFO:journalctl@ceph.node-exporter.b.smithi172.stdout:Apr 22 15:47:48 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@node-exporter.b.service: Consumed 3.172s CPU time. 2024-04-22T15:47:49.693 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:47:49 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=server t=2024-04-22T15:47:49.383522159Z level=info msg="Shutdown started" reason="System signal: terminated" 2024-04-22T15:47:49.693 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:47:49 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=ticker t=2024-04-22T15:47:49.383879022Z level=info msg=stopped last_tick=2024-04-22T15:47:40Z 2024-04-22T15:47:49.693 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:47:49 smithi172 ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a[63575]: logger=tracing t=2024-04-22T15:47:49.383900479Z level=info msg="Closing tracing" 2024-04-22T15:47:49.693 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:47:49 smithi172 podman[101583]: 2024-04-22 15:47:49.393856913 +0000 UTC m=+0.100670408 container died 2f825d15ab1081a1ab5ec9311e4549900b86622e6f3868cfc7bdf62250a400fc (image=quay.io/ceph/grafana:9.4.12, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-04-22T15:47:49.693 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:47:49 smithi172 podman[101583]: 2024-04-22 15:47:49.691295631 +0000 UTC m=+0.398109124 container cleanup 2f825d15ab1081a1ab5ec9311e4549900b86622e6f3868cfc7bdf62250a400fc (image=quay.io/ceph/grafana:9.4.12, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-04-22T15:47:49.967 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:47:49 smithi172 bash[101583]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a 2024-04-22T15:47:49.967 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:47:49 smithi172 podman[101597]: 2024-04-22 15:47:49.884186467 +0000 UTC m=+0.492085117 container remove 2f825d15ab1081a1ab5ec9311e4549900b86622e6f3868cfc7bdf62250a400fc (image=quay.io/ceph/grafana:9.4.12, name=ceph-869afcfa-00b9-11ef-bc93-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-04-22T15:47:50.439 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:47:50 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@grafana.a.service: Deactivated successfully. 2024-04-22T15:47:50.439 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:47:50 smithi172 systemd[1]: Stopped Ceph grafana.a for 869afcfa-00b9-11ef-bc93-c7b262605968. 2024-04-22T15:47:50.439 INFO:journalctl@ceph.grafana.a.smithi172.stdout:Apr 22 15:47:50 smithi172 systemd[1]: ceph-869afcfa-00b9-11ef-bc93-c7b262605968@grafana.a.service: Consumed 12.000s CPU time. 2024-04-22T15:48:01.739 DEBUG:teuthology.orchestra.run.smithi145:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-22T15:48:01.772 DEBUG:teuthology.orchestra.run.smithi172:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-22T15:48:01.798 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-22T15:48:01.799 DEBUG:teuthology.misc:Transferring archived files from smithi145:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/adking-2024-04-22_14:50:52-orch:cephadm-wip-adk-testing-2024-04-22-0809-distro-default-smithi/7668335/remote/smithi145/crash 2024-04-22T15:48:01.800 DEBUG:teuthology.orchestra.run.smithi145:> sudo tar c -f - -C /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/crash -- . 2024-04-22T15:48:01.856 INFO:teuthology.orchestra.run.smithi145.stderr:tar: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-22T15:48:01.856 INFO:teuthology.orchestra.run.smithi145.stderr:tar: Error is not recoverable: exiting now 2024-04-22T15:48:01.858 DEBUG:teuthology.misc:Transferring archived files from smithi172:/var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/adking-2024-04-22_14:50:52-orch:cephadm-wip-adk-testing-2024-04-22-0809-distro-default-smithi/7668335/remote/smithi172/crash 2024-04-22T15:48:01.859 DEBUG:teuthology.orchestra.run.smithi172:> sudo tar c -f - -C /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/crash -- . 2024-04-22T15:48:01.897 INFO:teuthology.orchestra.run.smithi172.stderr:tar: /var/lib/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-22T15:48:01.897 INFO:teuthology.orchestra.run.smithi172.stderr:tar: Error is not recoverable: exiting now 2024-04-22T15:48:01.898 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-22T15:48:01.898 DEBUG:teuthology.orchestra.run.smithi145:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-22T15:48:01.930 INFO:teuthology.orchestra.run.smithi145.stdout:2024-04-22T15:21:53.409400+0000 mon.a (mon.0) 1023 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] (CEPHADM_FAILED_DAEMON) 2024-04-22T15:48:01.930 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-04-22T15:48:01.930 DEBUG:teuthology.orchestra.run.smithi145:> sudo egrep '\[SEC\]' /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-22T15:48:01.997 DEBUG:teuthology.orchestra.run.smithi145:> sudo egrep '\[ERR\]' /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-22T15:48:02.067 DEBUG:teuthology.orchestra.run.smithi145:> sudo egrep '\[WRN\]' /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-22T15:48:02.134 INFO:teuthology.orchestra.run.smithi145.stdout:2024-04-22T15:21:53.409400+0000 mon.a (mon.0) 1023 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state'] (CEPHADM_FAILED_DAEMON) 2024-04-22T15:48:02.135 INFO:tasks.cephadm:Compressing logs... 2024-04-22T15:48:02.135 DEBUG:teuthology.orchestra.run.smithi145:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-22T15:48:02.177 DEBUG:teuthology.orchestra.run.smithi172:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-22T15:48:02.200 INFO:teuthology.orchestra.run.smithi145.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph.log 2024-04-22T15:48:02.200 INFO:teuthology.orchestra.run.smithi145.stderr:: No such file or directory 2024-04-22T15:48:02.200 INFO:teuthology.orchestra.run.smithi145.stderr:gzip -5 --verbose -- /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-osd.2.log 2024-04-22T15:48:02.200 INFO:teuthology.orchestra.run.smithi145.stderr:gzip -5 --verbose -- /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-22T15:48:02.200 INFO:teuthology.orchestra.run.smithi145.stderr:/var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph.log: /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-osd.3.log 2024-04-22T15:48:02.200 INFO:teuthology.orchestra.run.smithi145.stderr:/var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-osd.1.log 2024-04-22T15:48:02.201 INFO:teuthology.orchestra.run.smithi145.stderr:gzip -5 --verbose -- /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-mgr.y.log 2024-04-22T15:48:02.201 INFO:teuthology.orchestra.run.smithi145.stderr:/var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-osd.3.log: /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-osd.0.log 2024-04-22T15:48:02.201 INFO:teuthology.orchestra.run.smithi145.stderr:/var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-mon.c.log 2024-04-22T15:48:02.204 INFO:teuthology.orchestra.run.smithi172.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi172.glxwwa.log 2024-04-22T15:48:02.204 INFO:teuthology.orchestra.run.smithi172.stderr:: No such file or directory 2024-04-22T15:48:02.204 INFO:teuthology.orchestra.run.smithi172.stderr:gzip -5 --verbose -- /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph.log 2024-04-22T15:48:02.205 INFO:teuthology.orchestra.run.smithi172.stderr:gzip -5 --verbose -- /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-22T15:48:02.205 INFO:teuthology.orchestra.run.smithi172.stderr:/var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi172.glxwwa.log: /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-osd.7.log 2024-04-22T15:48:02.205 INFO:teuthology.orchestra.run.smithi172.stderr: 76.0%/var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph.audit.log: -- replaced with /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi172.glxwwa.log.gz 2024-04-22T15:48:02.205 INFO:teuthology.orchestra.run.smithi172.stderr:gzip -5 --verbose -- /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-osd.5.log 2024-04-22T15:48:02.205 INFO:teuthology.orchestra.run.smithi172.stderr:gzip -5 --verbose -- /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-osd.4.log 2024-04-22T15:48:02.205 INFO:teuthology.orchestra.run.smithi172.stderr:/var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-osd.7.log: /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-mgr.x.log 2024-04-22T15:48:02.205 INFO:teuthology.orchestra.run.smithi172.stderr:/var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-osd.6.log 2024-04-22T15:48:02.205 INFO:teuthology.orchestra.run.smithi172.stderr:/var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-22T15:48:02.205 INFO:teuthology.orchestra.run.smithi172.stderr:/var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-22T15:48:02.206 INFO:teuthology.orchestra.run.smithi172.stderr:/var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph.cephadm.log: 89.1% -- replaced with /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-22T15:48:02.207 INFO:teuthology.orchestra.run.smithi145.stderr:/var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-osd.0.log: 93.9% -- replaced with /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-22T15:48:02.207 INFO:teuthology.orchestra.run.smithi145.stderr:gzip -5 --verbose -- /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-mon.a.log 2024-04-22T15:48:02.208 INFO:teuthology.orchestra.run.smithi172.stderr: 96.0% -- replaced with /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-22T15:48:02.209 INFO:teuthology.orchestra.run.smithi172.stderr:gzip -5 --verbose -- /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-mon.b.log 2024-04-22T15:48:02.209 INFO:teuthology.orchestra.run.smithi172.stderr:/var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-volume.log: 91.4% -- replaced with /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-22T15:48:02.209 INFO:teuthology.orchestra.run.smithi172.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-22T15:48:02.211 INFO:teuthology.orchestra.run.smithi145.stderr:/var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-mon.c.log: 94.7% -- replaced with /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-22T15:48:02.223 INFO:teuthology.orchestra.run.smithi145.stderr:gzip -5 --verbose -- /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-22T15:48:02.224 INFO:teuthology.orchestra.run.smithi172.stderr:/var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-mon.b.log: /var/log/ceph/cephadm.log: 91.0% -- replaced with /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-22T15:48:02.231 INFO:teuthology.orchestra.run.smithi145.stderr:/var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/tcmu-runner.log 2024-04-22T15:48:02.239 INFO:teuthology.orchestra.run.smithi145.stderr:/var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph.cephadm.log: 97.1% -- replaced with /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-22T15:48:02.245 INFO:teuthology.orchestra.run.smithi172.stderr: 90.7% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-22T15:48:02.249 INFO:teuthology.orchestra.run.smithi145.stderr:gzip -5 --verbose -- /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-22T15:48:02.249 INFO:teuthology.orchestra.run.smithi145.stderr:/var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/tcmu-runner.log: 88.3% -- replaced with /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/tcmu-runner.log.gz 2024-04-22T15:48:02.257 INFO:teuthology.orchestra.run.smithi145.stderr:gzip -5 --verbose -- /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi145.ygrqmi.log 2024-04-22T15:48:02.282 INFO:teuthology.orchestra.run.smithi145.stderr:/var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-22T15:48:02.290 INFO:teuthology.orchestra.run.smithi145.stderr:/var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi145.ygrqmi.log: 76.0% -- replaced with /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi145.ygrqmi.log.gz 2024-04-22T15:48:02.298 INFO:teuthology.orchestra.run.smithi145.stderr:/var/log/ceph/cephadm.log: 91.0% -- replaced with /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-22T15:48:02.322 INFO:teuthology.orchestra.run.smithi172.stderr: 89.8% -- replaced with /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-mgr.x.log.gz 2024-04-22T15:48:02.327 INFO:teuthology.orchestra.run.smithi145.stderr: 92.5% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-22T15:48:02.740 INFO:teuthology.orchestra.run.smithi145.stderr: 89.0% -- replaced with /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-mgr.y.log.gz 2024-04-22T15:48:02.992 INFO:teuthology.orchestra.run.smithi145.stderr: 92.5% -- replaced with /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-mon.c.log.gz 2024-04-22T15:48:03.070 INFO:teuthology.orchestra.run.smithi172.stderr: 92.4% -- replaced with /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-mon.b.log.gz 2024-04-22T15:48:03.703 INFO:teuthology.orchestra.run.smithi172.stderr: 93.2% -- replaced with /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-osd.6.log.gz 2024-04-22T15:48:04.003 INFO:teuthology.orchestra.run.smithi145.stderr: 93.4% -- replaced with /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-osd.2.log.gz 2024-04-22T15:48:04.115 INFO:teuthology.orchestra.run.smithi172.stderr: 93.4% -- replaced with /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-osd.5.log.gz 2024-04-22T15:48:04.213 INFO:teuthology.orchestra.run.smithi145.stderr: 93.5% -- replaced with /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-osd.0.log.gz 2024-04-22T15:48:04.247 INFO:teuthology.orchestra.run.smithi172.stderr: 93.6% -- replaced with /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-osd.4.log.gz 2024-04-22T15:48:04.266 INFO:teuthology.orchestra.run.smithi145.stderr: 91.0% -- replaced with /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-mon.a.log.gz 2024-04-22T15:48:04.441 INFO:teuthology.orchestra.run.smithi172.stderr: 93.6% -- replaced with /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-osd.7.log.gz 2024-04-22T15:48:04.443 INFO:teuthology.orchestra.run.smithi172.stderr: 2024-04-22T15:48:04.443 INFO:teuthology.orchestra.run.smithi172.stderr:real 0m2.251s 2024-04-22T15:48:04.443 INFO:teuthology.orchestra.run.smithi172.stderr:user 0m8.398s 2024-04-22T15:48:04.443 INFO:teuthology.orchestra.run.smithi172.stderr:sys 0m0.357s 2024-04-22T15:48:04.464 INFO:teuthology.orchestra.run.smithi145.stderr: 93.4% -- replaced with /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-osd.1.log.gz 2024-04-22T15:48:04.618 INFO:teuthology.orchestra.run.smithi145.stderr: 93.4% -- replaced with /var/log/ceph/869afcfa-00b9-11ef-bc93-c7b262605968/ceph-osd.3.log.gz 2024-04-22T15:48:04.619 INFO:teuthology.orchestra.run.smithi145.stderr: 2024-04-22T15:48:04.619 INFO:teuthology.orchestra.run.smithi145.stderr:real 0m2.430s 2024-04-22T15:48:04.619 INFO:teuthology.orchestra.run.smithi145.stderr:user 0m11.422s 2024-04-22T15:48:04.619 INFO:teuthology.orchestra.run.smithi145.stderr:sys 0m0.447s 2024-04-22T15:48:04.620 INFO:tasks.cephadm:Archiving logs... 2024-04-22T15:48:04.620 DEBUG:teuthology.misc:Transferring archived files from smithi145:/var/log/ceph to /home/teuthworker/archive/adking-2024-04-22_14:50:52-orch:cephadm-wip-adk-testing-2024-04-22-0809-distro-default-smithi/7668335/remote/smithi145/log 2024-04-22T15:48:04.621 DEBUG:teuthology.orchestra.run.smithi145:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-22T15:48:06.180 DEBUG:teuthology.misc:Transferring archived files from smithi172:/var/log/ceph to /home/teuthworker/archive/adking-2024-04-22_14:50:52-orch:cephadm-wip-adk-testing-2024-04-22-0809-distro-default-smithi/7668335/remote/smithi172/log 2024-04-22T15:48:06.181 DEBUG:teuthology.orchestra.run.smithi172:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-22T15:48:07.437 INFO:tasks.cephadm:Removing cluster... 2024-04-22T15:48:07.437 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 --force 2024-04-22T15:48:07.728 DEBUG:teuthology.orchestra.run.smithi172:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 869afcfa-00b9-11ef-bc93-c7b262605968 --force 2024-04-22T15:48:08.018 INFO:tasks.cephadm:Removing cephadm ... 2024-04-22T15:48:08.019 DEBUG:teuthology.orchestra.run.smithi145:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-22T15:48:08.034 DEBUG:teuthology.orchestra.run.smithi172:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-22T15:48:08.049 INFO:tasks.cephadm:Teardown complete 2024-04-22T15:48:08.049 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-22T15:48:08.058 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-22T15:48:08.058 DEBUG:teuthology.orchestra.run.smithi145:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-22T15:48:08.076 DEBUG:teuthology.orchestra.run.smithi172:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-22T15:48:08.087 INFO:teuthology.orchestra.run.smithi145.stderr:bash: line 1: ntpq: command not found 2024-04-22T15:48:08.091 INFO:teuthology.orchestra.run.smithi145.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-22T15:48:08.091 INFO:teuthology.orchestra.run.smithi145.stdout:=============================================================================== 2024-04-22T15:48:08.091 INFO:teuthology.orchestra.run.smithi145.stdout:^- hv01.front.sepia.ceph.com 2 6 377 64 -1939us[-1932us] +/- 583ms 2024-04-22T15:48:08.092 INFO:teuthology.orchestra.run.smithi145.stdout:^* hv02.front.sepia.ceph.com 3 6 377 1 -147us[ -146us] +/- 68ms 2024-04-22T15:48:08.092 INFO:teuthology.orchestra.run.smithi145.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 63 +211us[ +218us] +/- 78ms 2024-04-22T15:48:08.092 INFO:teuthology.orchestra.run.smithi145.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T15:48:08.102 INFO:teuthology.orchestra.run.smithi172.stderr:bash: line 1: ntpq: command not found 2024-04-22T15:48:08.106 INFO:teuthology.orchestra.run.smithi172.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-22T15:48:08.106 INFO:teuthology.orchestra.run.smithi172.stdout:=============================================================================== 2024-04-22T15:48:08.106 INFO:teuthology.orchestra.run.smithi172.stdout:^- hv01.front.sepia.ceph.com 2 6 377 0 -1931us[-1931us] +/- 584ms 2024-04-22T15:48:08.106 INFO:teuthology.orchestra.run.smithi172.stdout:^* hv02.front.sepia.ceph.com 3 6 377 63 -180us[ -182us] +/- 67ms 2024-04-22T15:48:08.106 INFO:teuthology.orchestra.run.smithi172.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 1 +204us[ +204us] +/- 79ms 2024-04-22T15:48:08.106 INFO:teuthology.orchestra.run.smithi172.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T15:48:08.107 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-22T15:48:08.116 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-22T15:48:08.116 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-22T15:48:08.142 DEBUG:teuthology.orchestra.run.smithi145:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-22T15:48:08.229 DEBUG:teuthology.orchestra.run.smithi172:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-22T15:48:08.302 DEBUG:teuthology.orchestra.run.smithi145:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-22T15:48:08.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T15:48:08.330 DEBUG:teuthology.orchestra.run.smithi172:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-22T15:48:08.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T15:48:08.357 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-22T15:48:08.366 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-22T15:48:08.423 INFO:teuthology.task.internal:Duration was 2925.160945 seconds 2024-04-22T15:48:08.424 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-22T15:48:08.452 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-22T15:48:08.452 DEBUG:teuthology.orchestra.run.smithi145:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-22T15:48:08.455 DEBUG:teuthology.orchestra.run.smithi172:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-22T15:48:08.489 INFO:teuthology.orchestra.run.smithi145.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-22T15:48:08.491 INFO:teuthology.orchestra.run.smithi172.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-22T15:48:08.906 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-22T15:48:08.906 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi145.front.sepia.ceph.com 2024-04-22T15:48:08.906 DEBUG:teuthology.orchestra.run.smithi145:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-22T15:48:08.966 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi172.front.sepia.ceph.com 2024-04-22T15:48:08.966 DEBUG:teuthology.orchestra.run.smithi172:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-22T15:48:08.990 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-22T15:48:08.990 DEBUG:teuthology.orchestra.run.smithi145:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-22T15:48:09.008 DEBUG:teuthology.orchestra.run.smithi172:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-22T15:48:09.055 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-22T15:48:09.056 DEBUG:teuthology.orchestra.run.smithi145:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-22T15:48:09.073 DEBUG:teuthology.orchestra.run.smithi172:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-22T15:48:09.732 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-22T15:48:09.743 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-22T15:48:09.744 DEBUG:teuthology.orchestra.run.smithi145:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-22T15:48:09.766 DEBUG:teuthology.orchestra.run.smithi172:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-22T15:48:09.791 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-22T15:48:09.803 DEBUG:teuthology.orchestra.run.smithi145:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-22T15:48:09.809 DEBUG:teuthology.orchestra.run.smithi172:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-22T15:48:09.828 INFO:teuthology.orchestra.run.smithi145.stdout:kernel.core_pattern = core 2024-04-22T15:48:09.854 INFO:teuthology.orchestra.run.smithi172.stdout:kernel.core_pattern = core 2024-04-22T15:48:09.900 DEBUG:teuthology.orchestra.run.smithi145:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-22T15:48:09.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T15:48:09.916 DEBUG:teuthology.orchestra.run.smithi172:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-22T15:48:09.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T15:48:09.954 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-22T15:48:09.964 INFO:teuthology.task.internal:Transferring archived files... 2024-04-22T15:48:09.965 DEBUG:teuthology.misc:Transferring archived files from smithi145:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/adking-2024-04-22_14:50:52-orch:cephadm-wip-adk-testing-2024-04-22-0809-distro-default-smithi/7668335/remote/smithi145 2024-04-22T15:48:09.965 DEBUG:teuthology.orchestra.run.smithi145:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-22T15:48:10.017 DEBUG:teuthology.misc:Transferring archived files from smithi172:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/adking-2024-04-22_14:50:52-orch:cephadm-wip-adk-testing-2024-04-22-0809-distro-default-smithi/7668335/remote/smithi172 2024-04-22T15:48:10.018 DEBUG:teuthology.orchestra.run.smithi172:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-22T15:48:10.059 INFO:teuthology.task.internal:Removing archive directory... 2024-04-22T15:48:10.059 DEBUG:teuthology.orchestra.run.smithi145:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-22T15:48:10.062 DEBUG:teuthology.orchestra.run.smithi172:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-22T15:48:10.106 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-22T15:48:10.118 INFO:teuthology.task.internal:Not uploading archives. 2024-04-22T15:48:10.118 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-22T15:48:10.145 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-22T15:48:10.145 DEBUG:teuthology.orchestra.run.smithi145:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-22T15:48:10.148 DEBUG:teuthology.orchestra.run.smithi172:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-22T15:48:10.160 INFO:teuthology.orchestra.run.smithi145.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 22 15:48 /home/ubuntu/cephtest 2024-04-22T15:48:10.162 INFO:teuthology.orchestra.run.smithi172.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 22 15:48 /home/ubuntu/cephtest 2024-04-22T15:48:10.163 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-22T15:48:10.178 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-22T15:48:10.232 INFO:teuthology.nuke:Checking targets against current locks 2024-04-22T15:48:10.261 DEBUG:teuthology.nuke:shortname: smithi145 2024-04-22T15:48:10.261 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-22T15:48:10.287 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi145.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-04-22_14:50:52-orch:cephadm-wip-adk-testing-2024-04-22-0809-distro-default-smithi/7668335', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-22 14:52:15.246176', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHPdUOGkYO7/OwsFa+YW/UoxoMMcci4O3XA+snK5J2/1FxJyxa7DO4qcJo8QfCaF+BflZXltElTfrzhQxOjy2Ic='} 2024-04-22T15:48:10.288 DEBUG:teuthology.nuke:shortname: smithi172 2024-04-22T15:48:10.288 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-22T15:48:10.313 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi172.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-04-22_14:50:52-orch:cephadm-wip-adk-testing-2024-04-22-0809-distro-default-smithi/7668335', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-22 14:52:15.247885', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPkhKyWiTpQZhRqu8tjjYZYkldYw64E5tTU6Uq1IXcNeUeXTxPfFCXkMueOhIBCMRWNwKdNWDUh9zZJ0kdqZw6U='} 2024-04-22T15:48:10.338 INFO:teuthology.orchestra.console.smithi145:Power off 2024-04-22T15:48:10.339 DEBUG:teuthology.orchestra.console.smithi145:pexpect command: ipmitool -H smithi145.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-22T15:48:10.364 DEBUG:teuthology.orchestra.console.smithi145:power off output: Chassis Power Control: Down/Off 2024-04-22T15:48:10.365 DEBUG:teuthology.orchestra.console.smithi145:pexpect command: ipmitool -H smithi145.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-22T15:48:10.380 INFO:teuthology.orchestra.console.smithi172:Power off 2024-04-22T15:48:10.380 DEBUG:teuthology.orchestra.console.smithi172:pexpect command: ipmitool -H smithi172.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-22T15:48:10.389 DEBUG:teuthology.orchestra.console.smithi145:check power output: Chassis Power is on 2024-04-22T15:48:10.403 DEBUG:teuthology.orchestra.console.smithi172:power off output: Chassis Power Control: Down/Off 2024-04-22T15:48:10.403 DEBUG:teuthology.orchestra.console.smithi172:pexpect command: ipmitool -H smithi172.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-22T15:48:10.429 DEBUG:teuthology.orchestra.console.smithi172:check power output: Chassis Power is on 2024-04-22T15:48:14.390 DEBUG:teuthology.orchestra.console.smithi145:pexpect command: ipmitool -H smithi145.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-22T15:48:14.431 DEBUG:teuthology.orchestra.console.smithi172:pexpect command: ipmitool -H smithi172.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-22T15:48:14.500 DEBUG:teuthology.orchestra.console.smithi145:check power output: Chassis Power is on 2024-04-22T15:48:14.541 DEBUG:teuthology.orchestra.console.smithi172:check power output: Chassis Power is on 2024-04-22T15:48:18.502 DEBUG:teuthology.orchestra.console.smithi145:pexpect command: ipmitool -H smithi145.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-22T15:48:18.541 DEBUG:teuthology.orchestra.console.smithi172:pexpect command: ipmitool -H smithi172.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-22T15:48:18.612 DEBUG:teuthology.orchestra.console.smithi145:check power output: Chassis Power is off 2024-04-22T15:48:18.652 DEBUG:teuthology.orchestra.console.smithi172:check power output: Chassis Power is off 2024-04-22T15:48:18.714 INFO:teuthology.orchestra.console.smithi145:Power off completed 2024-04-22T15:48:18.753 INFO:teuthology.orchestra.console.smithi172:Power off completed 2024-04-22T15:48:18.859 INFO:teuthology.run:Summary data: description: orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} duration: 2925.160944700241 failure_reason: '"2024-04-22T15:21:53.409400+0000 mon.a (mon.0) 1023 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) [''daemon iscsi.foo.smithi145.ljpvqb on smithi145 is in unknown state''] (CEPHADM_FAILED_DAEMON)" in cluster log' owner: scheduled_adking@teuthology success: false 2024-04-22T15:48:18.860 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-22T15:48:18.943 INFO:teuthology.run:FAIL